Skip to content

Instantly share code, notes, and snippets.

/example.ts Secret

Created Jan 25, 2017
Embed
What would you like to do?
DRY Type Checked Encoders and Decoders in TypeScript 2.1 using Mapped Types
import util = require("util");
// DRY Type Checked Encoders and Decoders in TypeScript 2.1 using Mapped Types
// TL;DR: using new support for Mapped Types in TypeScript 2.1 we can build encoders and
// decoders, such as binary serializers, url routers and builders, and so on in a much safer
// and much more convenient way. If you want to play with this you'll need to use TypeScript 2.1.
namespace One {
// Suppose we have a type World
type World = {pings: number[];};
// and we have some data
const data: World = {pings: [1, 2, 3]};
// and suppose we want to encode and decode this data
const encoded = toBinary(data);
console.log(`encoded: ${util.inspect(encoded)}`);
console.log(`decoded: ${util.inspect(fromBinary(encoded))}`);
// Output:
// encoded: <Buffer ...>
// decoded: { pings: [ 1, 2, 3 ] }
// we'll write a function to turn our World into a Buffer
function toBinary(data: World): Buffer {
const buffer = Buffer.alloc(4 + data.pings.length * 8);
buffer.writeUInt32BE(data.pings.length, 0);
for (let i = 0; i < data.pings.length; ++i) {
buffer.writeDoubleBE(data.pings[i], 4 + i * 8);
}
return buffer;
}
// and a function to turn our Buffer back into a World
function fromBinary(buffer: Buffer): World {
const data: World = {pings: []}, length = buffer.readUInt32BE(0);
for (let i = 0; i < length; ++i) {
data.pings.push(buffer.readDoubleBE(4 + i * 8));
}
return data;
}
// easy, isn't it?
// Imagine what happens when we start changing data types or have to add more fields.
// Suddenly it becomes very difficult to maintain the encoder and decoder.
}
namespace Two {
// Suppose we have a type Entity, a type Message and a type World
type Entity = {id: number; actions: string[];};
type Message = {from: number; to: number; content: string};
type World = {entities: Entity[]; messages: Message[];};
// and we have some data.
const data: World = {
entities: [
{id: 0, actions: ["message sent"]},
{id: 1, actions: ["message sent"]}
],
messages: [
{from: 0, to: 1, content: "hello"},
{from: 1, to: 0, content: "welcome"}
]
};
// Imagine writing an encoder and decoder for the World type.
// It's not fun.
// Suppose we have functions for defining a format
const format = objectFormat({
entities: arrayFormat(objectFormat({
id: uint32Format(), actions: arrayFormat(stringFormat())
})),
messages: arrayFormat(objectFormat({
from: uint32Format(), to: uint32Format(), content: stringFormat()
}))
});
// and suppose we have functions that use this format to encode and decode our data.
const encoded = toBinary(data, format);
console.log(`encoded: ${util.inspect(encoded)}`);
console.log(`decoded: ${util.inspect(fromBinary(encoded, format), {depth: null})}`);
// Output:
// encoded: <Buffer ...>
// decoded: { entities:
// [ { actions: [ 'message sent' ], id: 0 },
// { actions: [ 'message sent' ], id: 1 } ],
// messages:
// [ { content: 'hello', from: 0, to: 1 },
// { content: 'welcome', from: 1, to: 0 } ] }
// Easy to refactor and entirely type checked.
// So how does this work?
// First up, the generic Format type
export interface Format<T> {
readonly dummy: T; // ignore this for now
// reader is a function that returns slices of the input buffer
decode(reader: (size: number) => Buffer): T;
// writer adds buffers to the output buffer
encode(value: T, writer: FormatWriter);
}
export interface FormatWriter {
(buffer: Buffer): void; // either by accepting an existing buffer
(size: number): Buffer; // or by creating a new buffer and returning it.
}
// We use these types to create formats for numbers, in this case doubles
export function float64Format(): Format<number> {
return numberFormat(8, (value, buffer) => buffer.writeDoubleBE(value, 0),
(buffer) => buffer.readDoubleBE(0));
}
// and unsigned integers
export function uint8Format(): Format<number> {
return numberFormat(1, (value, buffer) => buffer.writeUInt8(value, 0),
(buffer) => buffer.readUInt8(0));
}
export function uint32Format(): Format<number> {
return numberFormat(4, (value, buffer) => buffer.writeUInt32BE(value, 0),
(buffer) => buffer.readUInt32BE(0));
}
// a helper function cuts down on the boilerplate.
export function numberFormat(size: number, encode: (value: number, buffer: Buffer) => void,
decode: (buffer: Buffer) => number): Format<number> {
return {
dummy: null as number,
encode: (value, writer) => encode(value, writer(size)),
decode: (reader) => decode(reader(size))
}
}
// Next up: a format for strings
export function stringFormat(): Format<string> {
const lengthFormat = uint32Format();
return {
dummy: null as string,
encode: (value, writer) => {
const buffer = Buffer.from(value);
lengthFormat.encode(buffer.length, writer);
writer(buffer);
}, decode: (reader) => {
const length = lengthFormat.decode(reader);
return reader(length).toString();
}
};
}
// Next up: a format for arrays. This is where it gets tricky.
// Note how this function takes an element format of type T and returns a Format using the
// type of T's dummy element. This uses the new support for Lookup Types in TypeScript 2.1.
export function arrayFormat<T extends Format<any>>(format: T): Format<T["dummy"][]> {
const lengthFormat = uint32Format();
return {
dummy: null as T["dummy"][],
encode: (value, writer) => {
lengthFormat.encode(value.length, writer);
for (const element of value) {
format.encode(element, writer);
}
},
decode: (reader) => {
const elements: T["dummy"][] = [],
length = lengthFormat.decode(reader);
for (let i = 0; i < length; ++i) {
elements.push(format.decode(reader));
}
return elements;
}
};
}
// Next up: a format for objects.
// ObjectSpec is the supertype of arguments to objectFormat.
type ObjectSpec = {[key: string]: Format<any>};
// An ObjectValue is a value conforming to a subtype of ObjectSpec.
// This uses the new support for Mapped Types in TypeScript 2.1.
type ObjectValue<T extends ObjectSpec> = {[P in keyof T]: T[P]["dummy"];};
export function objectFormat<T extends ObjectSpec>(spec: T): Format<ObjectValue<T>> {
// First we transform our specification into an ordered list of items to prevent format
// differences should the output of Object.keys differ between JavaScript engines.
const items: [keyof T, Format<any>][] = [];
for (const key of Object.keys(spec).sort()) {
items.push([key, spec[key]]);
}
return {
dummy: null as ObjectValue<T>,
// Encoder and decoder are now straightforward.
encode: (value, writer) => {
for (const [key, format] of items) {
format.encode(value[key], writer);
}
},
decode: (reader) => {
const value = {} as ObjectValue<T>;
for (const [key, format] of items) {
value[key] = format.decode(reader);
}
return value;
}
}
}
// We finish by creating functions for encoding values using formats
export function toBinary<T>(value: T, format: Format<T>): Buffer {
const buffers: Buffer[] = [];
const writer = (buffer_or_size) => {
if (buffer_or_size instanceof Buffer) {
buffers.push(buffer_or_size);
} else {
const buffer = Buffer.alloc(buffer_or_size);
buffers.push(buffer);
return buffer;
}
};
format.encode(value, writer);
return Buffer.concat(buffers);
}
// and for decoding values using formats.
export function fromBinary<T>(buffer: Buffer, format: Format<T>): T {
let index = 0;
const reader = (size: number) => {
const slice = buffer.slice(index, index + size);
index += size;
return slice;
};
return format.decode(reader);
}
}
namespace Three {
import arrayFormat = Two.arrayFormat;
import Format = Two.Format;
import objectFormat = Two.objectFormat;
import toBinary = Two.toBinary;
import uint8Format = Two.uint8Format;
import float64Format = Two.float64Format;
// Let's see what we can do using our formats.
// For example, messaging for WebSockets. Let's define a type for message types
class MessageType<T> {
constructor(public readonly id: number, public readonly format: Format<T>) {
}
encode(value: T): Buffer {
return Buffer.concat([toBinary(this.id, MessageType.idFormat),
toBinary(value, this.format)])
}
static readonly idFormat = uint8Format();
}
// and one keeping track of ids
class MessageTypes {
private nextId = 0;
make<T extends Format<any>>(format: T): MessageType<T["dummy"]> {
return new MessageType(this.nextId++, format);
}
}
// and then a few types of messages.
const types = new MessageTypes(),
ping = types.make(float64Format()),
state = types.make(objectFormat({values: arrayFormat(float64Format())}));
// Let's define a type for handling encoded messages.
class MessageHandler {
private readonly handlers: {[key: number]: (reader: (size: number) => Buffer) => void}
= {};
on<T>(type: MessageType<T>, handler: (value: T) => void): this {
this.handlers[type.id] = (reader) => {
handler(type.format.decode(reader));
};
return this;
}
process(buffer: Buffer): boolean {
let index = 0;
const reader = (size: number): Buffer => {
const slice = buffer.slice(index, index + size);
index += size;
return slice;
};
try {
while (index < buffer.length) {
const id = MessageType.idFormat.decode(reader);
if (!(id in this.handlers)) return false;
this.handlers[id](reader);
}
} catch (e) {
return false;
}
return true;
}
}
// A handler using the types.
const handler = new MessageHandler()
.on(ping, (time) => {
console.log(`got ping: ${time}`);
})
.on(state, ({values}) => {
console.log(`got state: ${util.inspect(values)}`);
});
// Said handler handling a stream of messages.
handler.process(Buffer.concat([
ping.encode(257.123), state.encode({values: [9, 8, 7, 6, 5, 4.5]})
]));
// Output:
// got ping: 257.123
// got state: [ 9, 8, 7, 6, 5, 4.5 ]
}
namespace Four {
// And now for something slightly different. Suppose you want a type checked url builder and
// matcher. We will only implement a bare bones url builder here.
type FieldSpec = {[key: string]: Format<any>};
type FieldValue<T extends FieldSpec> = {[P in keyof T]: T[P]["_type"]};
class Format<T> {
readonly _type: T;
constructor(public readonly encode: (value: T) => string) {
}
}
class Route<T extends FieldSpec> {
private readonly parts: ((value: FieldValue<T>) => string)[] = [];
build(value: FieldValue<T>): string {
return "/" + this.parts.map((part) => part(value)).join("/");
}
field<U extends FieldSpec>(spec: U): Route<T & U> {
const next = new Route<T & U>(), keys = Object.keys(spec);
if (keys.length !== 1) throw "illegal number of fields";
const [key] = keys as [keyof U];
next.parts.push(...this.parts, (value) => spec[key].encode(value[key]));
return next;
}
path(path: string): Route<T> {
const next = new Route<T>();
next.parts.push(...this.parts, () => path);
return next;
}
}
const number = new Format<number>((value) => String(value)),
string = new Format<string>((value) => value);
console.log(`url: ${new Route().path("user").field({id: number}).path("contact")
.field({contact: string}).path("edit").build({id: 0, contact: "banana"})}`);
// Output:
// url: /user/0/contact/banana/edit
// Entirely type checked and tsserver is offering suggestions for fields in the build method.
// Of course, with nothing being perfect TypeScript allows us to re-use field names as long
// as the types are compatible. We also cannot enforce the constraint that one call to the
// field method must introduce one and exactly one new field using types.
}
// I recently discovered these techniques and I had to share them. I find them very elegant and
// much more convenient than writing encoders and decoders for complicated structures by hand.
// Questions, comments? Florian Junker <florian.junker@googlemail.com>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment