FastLargeFile.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Security;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Win32.SafeHandles;

namespace SaintCoinach.IO
{
    /// <summary>
    ///     A cheaty way to get very fast "Marshal.SizeOf" support without the overhead of the Marshaler each time.
    ///     Also provides a way to get the pointer of a generic type (useful for fast memcpy and other operations)
    /// </summary>
    /// <typeparam name="T"></typeparam>
    public static class SizeCache<T> where T : struct
    {
        /// <summary> The size of the Type </summary>
        public static readonly int Size;

        /// <summary> The real, underlying type. </summary>
        public static readonly Type Type;

        /// <summary> True if this type requires the Marshaler to map variables. (No direct pointer dereferencing) </summary>
        public static readonly bool TypeRequiresMarshal;

        internal static readonly GetUnsafePtrDelegate GetUnsafePtr;

        static SizeCache()
        {
            Type = typeof(T);
            // Bools = 1 char.
            if (typeof(T) == typeof(bool))
            {
                Size = 1;
            }
            else if (typeof(T).IsEnum)
            {
                Type = typeof(T).GetEnumUnderlyingType();
                Size = GetSizeOf(Type);
            }
            else
            {
                Size = GetSizeOf(Type);
            }

            TypeRequiresMarshal = GetRequiresMarshal(Type);

            // Generate a method to get the address of a generic type. We'll be using this for RtlMoveMemory later for much faster structure reads.
            var method = new DynamicMethod(string.Format("GetPinnedPtr<{0}>", typeof(T).FullName.Replace(".", "<>")),
                typeof(void*),
                new[] { typeof(T).MakeByRefType() },
                typeof(SizeCache<>).Module);

            ILGenerator generator = method.GetILGenerator();

            // ldarg 0
            generator.Emit(OpCodes.Ldarg_0);
            // (IntPtr)arg0
            generator.Emit(OpCodes.Conv_U);
            // ret arg0
            generator.Emit(OpCodes.Ret);
            GetUnsafePtr = (GetUnsafePtrDelegate)method.CreateDelegate(typeof(GetUnsafePtrDelegate));
        }

        private static int GetSizeOf(Type t)
        {
            try
            {
                // Try letting the marshaler handle getting the size.
                // It can *sometimes* do it correctly
                // If it can't, fall back to our own methods.
                var o = Activator.CreateInstance(t);
                return Marshal.SizeOf(o);
            }
            catch (Exception)
            {
                int totalSize = 0;
                var fields = t.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);

                foreach (var field in fields)
                {
                    var attr = field.GetCustomAttributes(typeof(FixedBufferAttribute), false);

                    if (attr.Length > 0)
                    {
                        var fba = (FixedBufferAttribute)attr[0];
                        totalSize += GetSizeOf(fba.ElementType) * fba.Length;
                        continue;
                    }

                    totalSize += GetSizeOf(field.FieldType);
                }
                return totalSize;
            }
        }

        private static bool GetRequiresMarshal(Type t)
        {
            var fields = t.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);
            foreach (var field in fields)
            {
                var requires = field.GetCustomAttributes(typeof(MarshalAsAttribute), true).Length != 0;

                if (requires)
                {
                    return true;
                }

                if (t == typeof(IntPtr))
                {
                    continue;
                }

                if (Type.GetTypeCode(t) == TypeCode.Object)
                {
                    requires |= GetRequiresMarshal(field.FieldType);
                }

                return requires;
            }
            return false;
        }

        #region Nested type: GetUnsafePtrDelegate

        internal unsafe delegate void* GetUnsafePtrDelegate(ref T value);

        #endregion
    }
    public static unsafe class BinaryReaderExtensionsEx
    {
        public static string ReadWString(this BinaryReader br, long pos = -1, bool returnToOrig = true)
        {
            if (pos == -1)
            {
                StringBuilder sb = new StringBuilder();
                ushort cur = br.ReadUInt16();
                while (cur != 0)
                {
                    sb.Append((char)cur);
                    cur = br.ReadUInt16();
                }
                return sb.ToString();
            }
            var orig = br.BaseStream.Position;
            br.BaseStream.Seek(pos, SeekOrigin.Begin);

            var s = ReadWString(br);

            if (returnToOrig)
            {
                br.BaseStream.Seek(orig, SeekOrigin.Begin);
            }

            return s;
        }

        /// <summary>
        /// Calls the native "memcpy" function.
        /// </summary>
        // Note: SuppressUnmanagedCodeSecurity speeds things up drastically since there is no stack-walk required before moving to native code.
        [DllImport("Kernel32.dll", EntryPoint = "RtlMoveMemory", SetLastError = false)]
        [SuppressUnmanagedCodeSecurity]
        internal static extern IntPtr MoveMemory(byte* dest, byte* src, int count);


        /// <summary>
        ///     Reads a generic structure from the current stream.
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="br"></param>
        /// <returns></returns>
        public static T Read<T>(this BinaryReader br) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException(
                    "Cannot read a generic structure type that requires marshaling support. Read the structure out manually.");
            }

            byte[] bytes = br.ReadBytes(SizeCache<T>.Size);
            if (bytes.Length < SizeCache<T>.Size)
                throw new EndOfStreamException();
            // OPTIMIZATION!
            var ret = new T();
            fixed (byte* b = bytes)
            {
                var tPtr = (byte*)SizeCache<T>.GetUnsafePtr(ref ret);
                MoveMemory(tPtr, b, SizeCache<T>.Size);
            }
            return ret;
        }

        public static T[] Read<T>(this BinaryReader br, long addr, long count) where T : struct
        {
            br.BaseStream.Seek(addr, SeekOrigin.Begin);
            return br.Read<T>(count);
        }

        public static T[] Read<T>(this BinaryReader br, long count) where T : struct
        {
            return br.Read<T>((int)count);
        }

        public static T[] Read<T>(this BinaryReader br, int count) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException("Cannot read a generic structure type that requires marshaling support. Read the structure out manually.");
            }

            if (count == 0)
            {
                return new T[0];
            }

            var ret = new T[count];
            fixed (byte* pB = br.ReadBytes(SizeCache<T>.Size * count))
            {
                var genericPtr = (byte*)SizeCache<T>.GetUnsafePtr(ref ret[0]);
                MoveMemory(genericPtr, pB, SizeCache<T>.Size * count);
            }
            return ret;
        }

        public static void Write<T>(this BinaryWriter bw, T value) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException(
                    "Cannot write a generic structure type that requires marshaling support. Write the structure out manually.");
            }

            // fastest way to copy?
            var buf = new byte[SizeCache<T>.Size];

            var valData = (byte*)SizeCache<T>.GetUnsafePtr(ref value);

            fixed (byte* pB = buf)
            {
                MoveMemory(pB, valData, SizeCache<T>.Size);
            }

            bw.Write(buf);
        }
    }
    /// <summary>
    /// A small wrapper around FileStream to help with large-file reading, without the overhead of .NET's internal contracts.
    /// </summary>
    public class FastLargeFile : IDisposable
    {
        private static readonly object ReadLock = new object();
        private readonly SafeFileHandle _handle;
        public long FileSize;
        private FileStream _stream;

        internal FileStream BaseStream { get { return _stream; } }

        public FastLargeFile(string fileName)
        {
            _stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
            _handle = _stream.SafeFileHandle;
            long unused;
            SetFilePointerEx(_handle, 0, out unused, 0);
            FileSize = _stream.Length;
        }

        #region IDisposable Members

        public void Dispose()
        {
            if (_stream != null)
            {
                _stream.Close();
                _stream = null;
            }
        }

        #endregion

        /// <summary>
        /// Calls the native "memcpy" function.
        /// </summary>
        // Note: SuppressUnmanagedCodeSecurity speeds things up drastically since there is no stack-walk required before moving to native code.
        [DllImport("Kernel32.dll", EntryPoint = "RtlMoveMemory", SetLastError = false)]
        [SuppressUnmanagedCodeSecurity]
        internal unsafe static extern IntPtr MoveMemory(byte* dest, byte* src, int count);

        [DllImport("kernel32.dll", SetLastError = true)]
        [return: MarshalAs(UnmanagedType.Bool)]
        private static extern bool ReadFile(SafeFileHandle handle, IntPtr buffer, uint numBytesToRead, out uint numBytesRead, IntPtr overlapped);

        [DllImport("kernel32.dll", SetLastError = true)]
        [return: MarshalAs(UnmanagedType.Bool)]
        private static extern bool SetFilePointerEx(SafeFileHandle hFile, long liDistanceToMove, out long lpNewFilePointer, uint dwMoveMethod);

        public BinaryReader GetReader(long offset, long count)
        {
            return new BinaryReader(new MemoryStream(ReadBytes(ref offset, count)));
        }


        public unsafe T ReadFast<T>(long offset) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException(
                    "Cannot read a generic structure type that requires marshaling support. Read the structure out manually.");
            }

            // OPTIMIZATION!
            var ret = new T();
            fixed (byte* b = ReadBytes(ref offset, SizeCache<T>.Size))
            {
                var tPtr = (byte*)SizeCache<T>.GetUnsafePtr(ref ret);
                MoveMemory(tPtr, b, SizeCache<T>.Size);
            }
            return ret;
        }


        public unsafe T ReadFast<T>(ref long offset) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException(
                    "Cannot read a generic structure type that requires marshaling support. Read the structure out manually.");
            }

            // OPTIMIZATION!
            var ret = new T();
            fixed (byte* b = ReadBytes(ref offset, SizeCache<T>.Size))
            {
                var tPtr = (byte*)SizeCache<T>.GetUnsafePtr(ref ret);
                MoveMemory(tPtr, b, SizeCache<T>.Size);
            }
            return ret;
        }


        public unsafe T[] ReadFast<T>(long offset, int count) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException(
                    "Cannot read a generic structure type that requires marshaling support. Read the structure out manually.");
            }

            if (count == 0)
            {
                return new T[0];
            }

            var ret = new T[count];
            fixed (byte* pB = ReadBytes(ref offset, SizeCache<T>.Size * count))
            {
                var genericPtr = (byte*)SizeCache<T>.GetUnsafePtr(ref ret[0]);
                MoveMemory(genericPtr, pB, SizeCache<T>.Size * count);
            }
            return ret;
        }

        public unsafe T[] ReadFast<T>(ref long offset, int count) where T : struct
        {
            if (SizeCache<T>.TypeRequiresMarshal)
            {
                throw new ArgumentException(
                    "Cannot read a generic structure type that requires marshaling support. Read the structure out manually.");
            }

            if (count == 0)
            {
                return new T[0];
            }

            var ret = new T[count];
            fixed (byte* pB = ReadBytes(ref offset, SizeCache<T>.Size * count))
            {
                var genericPtr = (byte*)SizeCache<T>.GetUnsafePtr(ref ret[0]);
                MoveMemory(genericPtr, pB, SizeCache<T>.Size * count);
            }
            return ret;
        }

        public unsafe byte[] ReadBytes(ref long offset, long count)
        {
            lock (ReadLock)
            {
                long unused;
                SetFilePointerEx(_handle, offset, out unused, 0);

                var buffer = new byte[count];
                fixed (byte* pFirst = &buffer[0])
                {
                    uint numRead;
                    if (ReadFile(_handle, (IntPtr)pFirst, (uint)count, out numRead, IntPtr.Zero) && numRead == count)
                    {
                        offset += count;
                        return buffer;
                    }
                }
                return null;
            }
        }

        ~FastLargeFile()
        {
            Dispose();
        }
    }
}