DRY Type Checked Encoders and Decoders in TypeScript 2.1 using Mapped Types

example.ts

import util = require("util");

// DRY Type Checked Encoders and Decoders in TypeScript 2.1 using Mapped Types

// TL;DR: using new support for Mapped Types in TypeScript 2.1 we can build encoders and
// decoders, such as binary serializers, url routers and builders, and so on in a much safer
// and much more convenient way. If you want to play with this you'll need to use TypeScript 2.1.

namespace One {
    // Suppose we have a type World
    type World = {pings: number[];};

    // and we have some data
    const data: World = {pings: [1, 2, 3]};

    // and suppose we want to encode and decode this data
    const encoded = toBinary(data);
    console.log(`encoded: ${util.inspect(encoded)}`);
    console.log(`decoded: ${util.inspect(fromBinary(encoded))}`);

    // Output:
    // encoded: <Buffer ...>
    // decoded: { pings: [ 1, 2, 3 ] }

    // we'll write a function to turn our World into a Buffer
    function toBinary(data: World): Buffer {
        const buffer = Buffer.alloc(4 + data.pings.length * 8);
        buffer.writeUInt32BE(data.pings.length, 0);
        for (let i = 0; i < data.pings.length; ++i) {
            buffer.writeDoubleBE(data.pings[i], 4 + i * 8);
        }
        return buffer;
    }

    // and a function to turn our Buffer back into a World
    function fromBinary(buffer: Buffer): World {
        const data: World = {pings: []}, length = buffer.readUInt32BE(0);
        for (let i = 0; i < length; ++i) {
            data.pings.push(buffer.readDoubleBE(4 + i * 8));
        }
        return data;
    }

    // easy, isn't it?

    // Imagine what happens when we start changing data types or have to add more fields.
    // Suddenly it becomes very difficult to maintain the encoder and decoder.
}

namespace Two {
    // Suppose we have a type Entity, a type Message and a type World
    type Entity = {id: number; actions: string[];};
    type Message = {from: number; to: number; content: string};
    type World = {entities: Entity[]; messages: Message[];};

    // and we have some data.
    const data: World = {
        entities: [
            {id: 0, actions: ["message sent"]},
            {id: 1, actions: ["message sent"]}
        ],
        messages: [
            {from: 0, to: 1, content: "hello"},
            {from: 1, to: 0, content: "welcome"}
        ]
    };

    // Imagine writing an encoder and decoder for the World type.
    // It's not fun.

    // Suppose we have functions for defining a format
    const format = objectFormat({
        entities: arrayFormat(objectFormat({
            id: uint32Format(), actions: arrayFormat(stringFormat())
        })),
        messages: arrayFormat(objectFormat({
            from: uint32Format(), to: uint32Format(), content: stringFormat()
        }))
    });

    // and suppose we have functions that use this format to encode and decode our data.
    const encoded = toBinary(data, format);
    console.log(`encoded: ${util.inspect(encoded)}`);
    console.log(`decoded: ${util.inspect(fromBinary(encoded, format), {depth: null})}`);

    // Output:
    // encoded: <Buffer ...>
    // decoded: { entities:
    //     [ { actions: [ 'message sent' ], id: 0 },
    //         { actions: [ 'message sent' ], id: 1 } ],
    //         messages:
    //     [ { content: 'hello', from: 0, to: 1 },
    //         { content: 'welcome', from: 1, to: 0 } ] }

    // Easy to refactor and entirely type checked.

    // So how does this work?
    // First up, the generic Format type
    export interface Format<T> {
        readonly dummy: T; // ignore this for now

        // reader is a function that returns slices of the input buffer
        decode(reader: (size: number) => Buffer): T;

        // writer adds buffers to the output buffer
        encode(value: T, writer: FormatWriter);
    }

    export interface FormatWriter {
        (buffer: Buffer): void; // either by accepting an existing buffer
        (size: number): Buffer; // or by creating a new buffer and returning it.
    }

    // We use these types to create formats for numbers, in this case doubles
    export function float64Format(): Format<number> {
        return numberFormat(8, (value, buffer) => buffer.writeDoubleBE(value, 0),
            (buffer) => buffer.readDoubleBE(0));
    }

    // and unsigned integers
    export function uint8Format(): Format<number> {
        return numberFormat(1, (value, buffer) => buffer.writeUInt8(value, 0),
            (buffer) => buffer.readUInt8(0));
    }

    export function uint32Format(): Format<number> {
        return numberFormat(4, (value, buffer) => buffer.writeUInt32BE(value, 0),
            (buffer) => buffer.readUInt32BE(0));
    }

    // a helper function cuts down on the boilerplate.
    export function numberFormat(size: number, encode: (value: number, buffer: Buffer) => void,
                                 decode: (buffer: Buffer) => number): Format<number> {
        return {
            dummy: null as number,
            encode: (value, writer) => encode(value, writer(size)),
            decode: (reader) => decode(reader(size))
        }
    }

    // Next up: a format for strings
    export function stringFormat(): Format<string> {
        const lengthFormat = uint32Format();
        return {
            dummy: null as string,
            encode: (value, writer) => {
                const buffer = Buffer.from(value);
                lengthFormat.encode(buffer.length, writer);
                writer(buffer);
            }, decode: (reader) => {
                const length = lengthFormat.decode(reader);
                return reader(length).toString();
            }
        };
    }

    // Next up: a format for arrays. This is where it gets tricky.
    // Note how this function takes an element format of type T and returns a Format using the
    // type of T's dummy element. This uses the new support for Lookup Types in TypeScript 2.1.
    export function arrayFormat<T extends Format<any>>(format: T): Format<T["dummy"][]> {
        const lengthFormat = uint32Format();
        return {
            dummy: null as T["dummy"][],
            encode: (value, writer) => {
                lengthFormat.encode(value.length, writer);
                for (const element of value) {
                    format.encode(element, writer);
                }
            },
            decode: (reader) => {
                const elements: T["dummy"][] = [],
                    length = lengthFormat.decode(reader);
                for (let i = 0; i < length; ++i) {
                    elements.push(format.decode(reader));
                }
                return elements;
            }
        };
    }

    // Next up: a format for objects.
    // ObjectSpec is the supertype of arguments to objectFormat.
    type ObjectSpec = {[key: string]: Format<any>};

    // An ObjectValue is a value conforming to a subtype of ObjectSpec.
    // This uses the new support for Mapped Types in TypeScript 2.1.
    type ObjectValue<T extends ObjectSpec> = {[P in keyof T]: T[P]["dummy"];};

    export function objectFormat<T extends ObjectSpec>(spec: T): Format<ObjectValue<T>> {
        // First we transform our specification into an ordered list of items to prevent format
        // differences should the output of Object.keys differ between JavaScript engines.
        const items: [keyof T, Format<any>][] = [];
        for (const key of Object.keys(spec).sort()) {
            items.push([key, spec[key]]);
        }
        return {
            dummy: null as ObjectValue<T>,
            // Encoder and decoder are now straightforward.
            encode: (value, writer) => {
                for (const [key, format] of items) {
                    format.encode(value[key], writer);
                }
            },
            decode: (reader) => {
                const value = {} as ObjectValue<T>;
                for (const [key, format] of items) {
                    value[key] = format.decode(reader);
                }
                return value;
            }
        }
    }

    // We finish by creating functions for encoding values using formats
    export function toBinary<T>(value: T, format: Format<T>): Buffer {
        const buffers: Buffer[] = [];
        const writer = (buffer_or_size) => {
            if (buffer_or_size instanceof Buffer) {
                buffers.push(buffer_or_size);
            } else {
                const buffer = Buffer.alloc(buffer_or_size);
                buffers.push(buffer);
                return buffer;
            }
        };
        format.encode(value, writer);
        return Buffer.concat(buffers);
    }

    // and for decoding values using formats.
    export function fromBinary<T>(buffer: Buffer, format: Format<T>): T {
        let index = 0;
        const reader = (size: number) => {
            const slice = buffer.slice(index, index + size);
            index += size;
            return slice;
        };
        return format.decode(reader);
    }
}

namespace Three {
    import arrayFormat = Two.arrayFormat;
    import Format = Two.Format;
    import objectFormat = Two.objectFormat;
    import toBinary = Two.toBinary;
    import uint8Format = Two.uint8Format;
    import float64Format = Two.float64Format;

    // Let's see what we can do using our formats.

    // For example, messaging for WebSockets. Let's define a type for message types
    class MessageType<T> {
        constructor(public readonly id: number, public readonly format: Format<T>) {
        }

        encode(value: T): Buffer {
            return Buffer.concat([toBinary(this.id, MessageType.idFormat),
                toBinary(value, this.format)])
        }

        static readonly idFormat = uint8Format();
    }

    // and one keeping track of ids
    class MessageTypes {
        private nextId = 0;

        make<T extends Format<any>>(format: T): MessageType<T["dummy"]> {
            return new MessageType(this.nextId++, format);
        }
    }

    // and then a few types of messages.
    const types = new MessageTypes(),
        ping = types.make(float64Format()),
        state = types.make(objectFormat({values: arrayFormat(float64Format())}));

    // Let's define a type for handling encoded messages.
    class MessageHandler {
        private readonly handlers: {[key: number]: (reader: (size: number) => Buffer) => void}
            = {};

        on<T>(type: MessageType<T>, handler: (value: T) => void): this {
            this.handlers[type.id] = (reader) => {
                handler(type.format.decode(reader));
            };
            return this;
        }

        process(buffer: Buffer): boolean {
            let index = 0;
            const reader = (size: number): Buffer => {
                const slice = buffer.slice(index, index + size);
                index += size;
                return slice;
            };
            try {
                while (index < buffer.length) {
                    const id = MessageType.idFormat.decode(reader);
                    if (!(id in this.handlers)) return false;
                    this.handlers[id](reader);
                }
            } catch (e) {
                return false;
            }
            return true;
        }
    }

    // A handler using the types.
    const handler = new MessageHandler()
        .on(ping, (time) => {
            console.log(`got ping: ${time}`);
        })
        .on(state, ({values}) => {
            console.log(`got state: ${util.inspect(values)}`);
        });

    // Said handler handling a stream of messages.
    handler.process(Buffer.concat([
        ping.encode(257.123), state.encode({values: [9, 8, 7, 6, 5, 4.5]})
    ]));

    // Output:
    // got ping: 257.123
    // got state: [ 9, 8, 7, 6, 5, 4.5 ]
}

namespace Four {
    // And now for something slightly different. Suppose you want a type checked url builder and
    // matcher. We will only implement a bare bones url builder here.

    type FieldSpec = {[key: string]: Format<any>};

    type FieldValue<T extends FieldSpec> = {[P in keyof T]: T[P]["_type"]};

    class Format<T> {
        readonly _type: T;

        constructor(public readonly encode: (value: T) => string) {
        }
    }

    class Route<T extends FieldSpec> {
        private readonly parts: ((value: FieldValue<T>) => string)[] = [];

        build(value: FieldValue<T>): string {
            return "/" + this.parts.map((part) => part(value)).join("/");
        }

        field<U extends FieldSpec>(spec: U): Route<T & U> {
            const next = new Route<T & U>(), keys = Object.keys(spec);
            if (keys.length !== 1) throw "illegal number of fields";
            const [key] = keys as [keyof U];
            next.parts.push(...this.parts, (value) => spec[key].encode(value[key]));
            return next;
        }

        path(path: string): Route<T> {
            const next = new Route<T>();
            next.parts.push(...this.parts, () => path);
            return next;
        }
    }

    const number = new Format<number>((value) => String(value)),
        string = new Format<string>((value) => value);

    console.log(`url: ${new Route().path("user").field({id: number}).path("contact")
        .field({contact: string}).path("edit").build({id: 0, contact: "banana"})}`);

    // Output:
    // url: /user/0/contact/banana/edit

    // Entirely type checked and tsserver is offering suggestions for fields in the build method.
    // Of course, with nothing being perfect TypeScript allows us to re-use field names as long
    // as the types are compatible. We also cannot enforce the constraint that one call to the
    // field method must introduce one and exactly one new field using types.
}

// I recently discovered these techniques and I had to share them. I find them very elegant and
// much more convenient than writing encoders and decoders for complicated structures by hand.

// Questions, comments? Florian Junker <florian.junker@googlemail.com>