010 Editor Template for dotnet's Binary Format [MS-NRBF]

binary-format-template.md

Binary Format Template

An 010 Editor template for the binary format used by dotnet's BinaryFormatter.

Full spec: https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-nrbf/75b9fe09-be15-475f-85b8-ae7b7558cfe5

Not fully complete but should work for most structures

Also included is example data and the script used to generate it. Requires dotnet script

binaryformat.bt

//------------------------------------------------
//--- 010 Editor v11.0.1 Binary Template
//
//   Authors: Camas
//   Purpose: Read dotnet's Binary Format
//------------------------------------------------

// See: https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-nrbf/75b9fe09-be15-475f-85b8-ae7b7558cfe5

// Rust-like typedefs as they're easier to understand
typedef ubyte bool;
typedef ubyte u8;
typedef byte i8;
typedef ushort u16;
typedef short i16;
typedef uint u32;
typedef int i32;
typedef uquad u64;
typedef quad i64;
typedef float f32;
typedef double f64;

// Helper functions

void todo() {
    Printf("TODO: Not implemented yet\n");
    Exit(-1);
}

// Forward Declarations

struct MemberValues;
struct BinaryArrayValues;
struct Record;

// 2.1 Common Definitions

// 2.1.1 Common Data Types

// LEB128 encoded i32. Used by Read7BitEncodedInt
typedef struct {
    local i32 value = 0;
    local i32 shift = 0;
    while (true) {
        u8 b;
        value |= (b & 0x7F) << shift;
        if ((b & 0x80) == 0) {
            break;
        }
        shift += 7;
        if (shift >= 35) {
            Warning("encoded_i32 should not be longer than 5 bytes! Skipping remaining");
            break;
        }
    }
} encoded_i32 <read=read_encoded_i32>;

string read_encoded_i32(encoded_i32 &v) {
    local string s;
    SPrintf( s, "%d", v.value );
    return s;
};

typedef struct {
    encoded_i32 length;
    if (length.value > 0) {
        char value[length.value]<optimize=false, fgcolor=0xffbbff>;
    } else {
        local string value = "";
    }
} String<read=read_String>;

string read_String(String &s) {
    return s.value;
}

typedef struct TimeSpan {
    i64 nanoseconds;
};

typedef struct DateTime {
    i64 ticks : 62;
    u8 kind: 2;
};

typedef struct Decimal {
    String value;
};

typedef struct ClassTypeInfo {
    String type_name;
    i32 library_id;
};

// 2.1.2 Enumerations

enum <u8> RecordType {
    RecordType_SerializedStreamHeader,
    RecordType_ClassWithId,
    RecordType_SystemClassWithMembers,
    RecordType_ClassWithMembers,
    RecordType_SystemClassWithMembersAndTypes,
    RecordType_ClassWithMembersAndTypes,
    RecordType_BinaryObjectString,
    RecordType_BinaryArray,
    RecordType_MemberPrimitiveTyped,
    RecordType_MemberReference,
    RecordType_ObjectNull,
    RecordType_MessageEnd,
    RecordType_BinaryLibrary,
    RecordType_ObjectNullMultiple256,
    RecordType_ObjectNullMultiple,
    RecordType_ArraySinglePrimitive,
    RecordType_ArraySingleObject,
    RecordType_ArraySingleString,
    RecordType_MethodCall = 21,
    RecordType_MethodReturn,
};

enum <u8> BinaryType {
    Primitive,
    BinaryType_String,
    Object,
    SystemClass,
    Class,
    ObjectArray,
    StringArray,
    PrimitiveArray,
};

enum <u8> PrimitiveType {
    Boolean = 1,
    Byte,
    Char,
    PrimitiveType_Decimal = 5,
    Double,
    Int16,
    Int32,
    Int64,
    SByte,
    Single,
    PrimitiveType_TimeSpan,
    PrimitiveType_DateTime,
    UInt16,
    UInt32,
    UInt64,
    Null,
    PrimitiveType_String,
};

typedef struct PrimitiveTypeValue (PrimitiveType type) {
    switch (type) {
        case Boolean:
            bool value<fgcolor=0xff8888>;
            break;
        case Byte:
            u8 value<fgcolor=0xff8888>;
            break;
        case Char:
            char value<fgcolor=0xff8888>;
            break;
        case PrimitiveType_Decimal:
            Decimal value<fgcolor=0xff8888>;
            break;
        case Double:
            f64 value<fgcolor=0xff8888>;
            break;
        case Int16:
            i16 value<fgcolor=0xff8888>;
            break;
        case Int32:
            i32 value<fgcolor=0xff8888>;
            break;
        case Int64:
            i64 value<fgcolor=0xff8888>;
            break;
        case SByte:
            i8 value<fgcolor=0xff8888>;
            break;
        case Single:
            f32 value<fgcolor=0xff8888>;
            break;
        case PrimitiveType_TimeSpan:
            TimeSpan value<fgcolor=0xff8888>;
            break;
        case PrimitiveType_DateTime:
            DateTime value<fgcolor=0xff8888>;
            break;
        case UInt16:
            u16 value<fgcolor=0xff8888>;
            break;
        case UInt32:
            u32 value<fgcolor=0xff8888>;
            break;
        case UInt64:
            u64 value<fgcolor=0xff8888>;
            break;
        case Null:
            break;
        case PrimitiveType_String:
            String value<fgcolor=0xff8888>;
            break;
    }
};

// 2.2 Method Invocation Records

// 2.2.1 Enumerations

typedef struct MessageFlags {
    bool NoArgs : 1;
    bool ArgsInline : 1;
    bool ArgsIsArray : 1;
    bool ArgsInArray : 1;
    bool NoContext : 1;
    bool ContextInline : 1;
    bool ContextInArray : 1;
    bool MethodSignatureInArray : 1;
    bool PropertiesInArray : 1;
    bool NoReturnValue : 1;
    bool ReturnValueVoid : 1;
    bool ReturnValueInline : 1;
    bool ReturnValueInArray : 1;
    bool ExceptionInArray : 1;
    bool _unused1 : 1;
    bool GenericMethod : 1;
    i16 _unused2;
};

// 2.2.2 Common Structures

typedef struct ValueWithCode {
    PrimitiveType type;
    PrimitiveTypeValue value;
};

typedef struct {
    ValueWithCode inner;
    Assert(inner.type == PrimitiveType_String);
} StringValueWithCode<read=StringValueWithCode_read>;

string StringValueWithCode_read(StringValueWithCode &s) {
    return s.value.value;
}

typedef struct ArrayOfValueWithCode {
    i32 length;
    ValueWithCode values[length]<optimize=false>;
};

// 2.2.3 Record Definitions

typedef struct BinaryMethodCall {
    MessageFlags message_enum;
    StringValueWithCode method_name;
    StringValueWithCode type_name;
    StringValueWithCode call_context;
    ArrayOfValueWithCode args;
};

typedef struct BinaryMethodReturn {
    MessageFlags message_enum;
    ValueWithCode return_value;
    StringValueWithCode call_context;
    ArrayOfValueWithCode args;
};

// 2.3 Class Records

// 2.3.1 Common Structures

typedef struct ClassInfo {
    i32 object_id;
    String name;
    i32 member_count;
    String member_names[member_count]<optimize=false>;
};

typedef struct MemberTypeInfo (i32 member_count) {
    local i32 count = member_count;
    BinaryType binary_type_enums[member_count]<optimize=false>;
    local i32 i;
    local BinaryType t;
    for (i = 0; i < member_count; i++) {
        t = binary_type_enums[i];
        switch (t) {
            case Primitive:
                PrimitiveType info;
                break;
            case SystemClass:
                String info;
                break;
            case Class:
                ClassTypeInfo info;
                break;
            case PrimitiveArray:
                PrimitiveType info;
                break;
            default:
                break;
        }
    }
};

// 2.3.2 Record Definitions

typedef struct ClassWithMembersAndTypes {
    ClassInfo class_info;
    MemberTypeInfo member_type_info(class_info.member_count);
    i32 library_id;
    MemberValues members(member_type_info);
};

typedef struct ClassWithMembers {
    ClassInfo class_info;
    i32 library_id;
    todo();
};

typedef struct SystemClassWithMembersAndTypes {
    ClassInfo class_info;
    MemberTypeInfo member_type_info(class_info.member_count);
    todo();
};

typedef struct SystemClassWithMembers {
    ClassInfo class_info;
    todo();
};

typedef struct ClassWithId {
    i32 object_id;
    i32 metadata_id;
    todo();
};

// 2.4 Array Records

// 2.4.1 Enumerations

enum <u8> BinaryArrayType {
    BinaryArrayType_Single,
    Jagged,
    Rectangular,
    SingleOffset,
    JaggedOffset,
    RectangularOffset,
};

// 2.4.2 Common Definitions

typedef struct ArrayInfo {
    i32 object_id;
    i32 length;
};

// 2.4.3 Record Definitions

typedef struct BinaryArray {
    i32 object_id;
    BinaryArrayType binary_array_type;
    i32 rank;
    i32 lengths[rank];
    if (binary_array_type == SingleOffset || binary_array_type == JaggedOffset || binary_array_type == RectangularOffset) {
        i32 lower_bounds[rank];
    }
    BinaryType type_enum;
    switch (type_enum) {
        case Primitive:
            PrimitiveType info;
            break;
        case SystemClass:
            String info;
            break;
        case Class:
            ClassTypeInfo info;
            break;
        case PrimitiveArray:
            PrimitiveType info;
        default:
            break;
    }

    // Values
    local u64 count = 0;
    if (rank > 0) {
        count = 1;
        local i32 i;
        for (i = 0; i < rank; i++) {
            count *= lengths[i];
        }
    }
    switch (type_enum) {
        case Primitive:
            PrimitiveTypeValue values(info)[count]<optimize=false>;
            break;
        case BinaryType_String:
        case SystemClass:
        case Class:
        case ObjectArray:
        case StringArray:
        case PrimitiveArray:
            Record values[count]<optimize=false>;
            break;
        default:
            Warning("Unhandled binary type");
            todo();
    }
};

typedef struct ArraySingleObject {
    ArrayInfo array_info;
    todo();
};

typedef struct ArraySinglePrimitive {
    ArrayInfo array_info;
    PrimitiveType primitive_type;
    PrimitiveTypeValue values(primitive_type)[array_info.length]<optimize=false>;
};

typedef struct ArraySingleString {
    ArrayInfo array_info;
    todo();
};

// 2.5 Member Reference Records

typedef struct MemberPrimitiveTyped {
    PrimitiveType type;
    PrimitiveTypeValue value(type);
};

typedef struct MemberPrimitiveUnTyped(PrimitiveType type) {
    PrimitiveTypeValue value(type);
};

typedef struct MemberReference {
    i32 id_ref;
};

typedef struct ObjectNull {};

typedef struct ObjectNullMultiple {
    i32 null_count;
};

typedef struct ObjectNullMultiple256 {
    u8 null_count;
};

typedef struct BinaryObjectString {
    i32 object_id;
    String value;
};

// 2.6 Other Records

typedef struct SerializationHeader {
    i32 root_id;
    i32 header_id;
    i32 major_version;
    i32 minor_version;
};

typedef struct BinaryLibrary {
    i32 library_id;
    String library_name;
};

typedef struct MessageEnd {};

// Helper structs

typedef struct Record {
    RecordType type_enum<fgcolor=0x44ff44>;
    switch (type_enum) {
        case RecordType_SerializedStreamHeader:
            SerializationHeader record;
            break;
        case RecordType_ClassWithId:
            ClassWithId record;
            break;
        case RecordType_SystemClassWithMembers:
            SystemClassWithMembers record;
            break;
        case RecordType_ClassWithMembers:
            ClassWithMembers record;
            break;
        case RecordType_SystemClassWithMembersAndTypes:
            SystemClassWithMembersAndTypes record;
            MemberValues members(record.member_type_info);
            break;
        case RecordType_ClassWithMembersAndTypes:
            ClassWithMembersAndTypes record;
            break;
        case RecordType_BinaryObjectString:
            BinaryObjectString record;
            break;
        case RecordType_BinaryArray:
            BinaryArray record;
            break;
        case RecordType_MemberPrimitiveTyped:
            MemberPrimitiveTyped record;
            break;
        case RecordType_MemberReference:
            MemberReference record;
            break;
        case RecordType_ObjectNull:
            ObjectNull record;
            break;
        case RecordType_MessageEnd:
            MessageEnd record;
            break;
        case RecordType_BinaryLibrary:
            BinaryLibrary record;
            break;
        case RecordType_ObjectNullMultiple256:
            ObjectNullMultiple256 record;
            break;
        case RecordType_ObjectNullMultiple:
            ObjectNullMultiple record;
            break;
        case RecordType_ArraySinglePrimitive:
            ArraySinglePrimitive record;
            break;
        case RecordType_ArraySingleObject:
            ArraySingleObject record;
            break;
        case RecordType_ArraySingleString:
            ArraySingleString record;
            break;
        case RecordType_MethodCall:
            BinaryMethodCall record;
            break;
        case RecordType_MethodReturn:
            BinaryMethodReturn record;
            break;
        default:
            Warning("Unknown record type %d", type_enum);
    }
};

typedef struct MemberValues(MemberTypeInfo &info) {
    local PrimitiveType p_info;
    // local ClassTypeInfo c_info;
    local BinaryType t;
    local string s_info;
    local i32 j = 0;
    local i32 i;
    for (i = 0; i < info.count; i++) {
        t = info.binary_type_enums[i];
        switch (t) {
            case Primitive:
                p_info = info.info[j];
                j++;
                PrimitiveTypeValue value(p_info);
                break;
            case BinaryType_String:
                Record value;
                break;
            case SystemClass:
                // s_info = info.info[j].value;
                j++;
                Record value;
                break;
            case Class:
                // c_info = info.info[j];
                j++;
                Record value;
                break;
            case ObjectArray:
                Record value;
                break;
            case StringArray:
                Record value;
                break;
            case PrimitiveArray:
                // p_info = info.info[j];
                j++;
                Record value;
                break;
            default:
                Warning("Unhandled binary type");
                todo();
        }
    }
};

// Read Records

while (!FEof()) {
    Record record;
}

gen-binary.csx

using System.Dynamic;
using System.Linq.Expressions;
using System.Runtime.Serialization.Formatters.Binary;

// `$ dotnet script gen-binary.csx`

[Serializable]
public class A
{
    public string SomeString { get; set; } = "abc";
    public int SomeValue { get; set; } = 125;
    public bool bool1 = true;
    public double DoubleValueWithLongName = 12.01234d;
    public long? nullableWithValue = 11902873498723094;
    public int? nullable = null;
    public int[] array = { 12, 34, 1 };
    public int[,] dimArray = { { 1, 2, 3 }, { 4, 5, 6 } };
    public int[][] jaggedArray = { new int[] { 1 }, new int[] { 2, 3 }, new int[] { 4, 5, 6 }, new int[] { 99999 } };
    public B InnerClassB = new B();
    [Serializable]
    public class B
    {
        public int ID = 891;
    }
}

BinaryFormatter bf = new BinaryFormatter();
StreamWriter sw = new StreamWriter("test.bin");
bf.Serialize(sw.BaseStream, new A());
sw.Close();

screenshot.png

test.bin