Reversal of mIRC rich text (ASCII only)

mircrev.c

#include <ctype.h>

#define BIT(X)  (1 << (X))
#define CTRL(X) ((X) - '@')

enum { None = -1 };

enum {
	Bold   = CTRL('B'),
	Colour = CTRL('C'),
	Reset  = CTRL('O'),
	Video  = CTRL('V'),
};

struct state {
	int colour;
	int flags;
};

static char *emitflags(char *, int);
static char *update(char *, struct state, struct state);

char *
mircrev(char *p, const char *s)
{
	char c;
	struct state rhs = { .colour = None, .flags = 0 };
	struct state lhs = rhs;

	*p = '\0';

	do {
		switch ((c = *s++)) {
		case Bold:
		case Video:
			lhs.flags ^= BIT(c); // toggle bit
			break;
		case Colour:
			if (isdigit(*s)) {
				lhs.colour = *s++ - '0';
				if (isdigit(*s)) {
					lhs.colour = (lhs.colour * 10) + (*s++ - '0');
				}
			}
			else {
				lhs.colour = None;
			}
			break;
		case '\0':
		case Reset:
			lhs.flags = 0; // turn all flags off
			break;
		default:
			p = update(p, lhs, rhs);
			rhs = lhs;
			*--p = c;
			break;
		}
	} while (c);

	return update(p, lhs, rhs);
}

char *
emitflags(char *p, int flags)
{
	if ((flags & BIT(Bold))) {
		*--p = Bold;
	}
	if ((flags & BIT(Video))) {
		*--p = Video;
	}
	return p;
}

char *
update(char *p, struct state lhs, struct state rhs)
{
	if (rhs.colour == lhs.colour && rhs.flags == lhs.flags) {
		// Nothing has changed.
		return p;
	}

	if (*p == '\0') {
		// No need to update at end of string
		return p;
	}

	if (rhs.colour == None && rhs.flags == 0) {
		// Everything will be turned off, so do a full reset.
		*--p = Reset;
		return p;
	}

	int delta = lhs.flags ^ rhs.flags;

	if (!delta && rhs.colour == None && lhs.colour != None && isdigit(*p)) {
		// There is a trailing digit which would be read as part of a colour
		// code, so we have to disambiguate it somehow.
		if ((rhs.flags & (rhs.flags - 1)) == 0) {
			// Only one flag will be left on, so the cheapest way is to do a
			// reset and then turn the flag back on again.
			p = emitflags(p, rhs.flags);
			*--p = Reset;
		}
		else {
			// Toggle bold so as to separate colour switch from digit.
			*--p = Bold;
			*--p = Bold;
			*--p = Colour;
		}
		return p;
	}

	p = emitflags(p, delta);

	if (rhs.colour != lhs.colour) {
		int digit = isdigit(*p);

		if (rhs.colour != None) {
			// Update to rhs colour value.
			*--p = '0' + (rhs.colour % 10);
			if (rhs.colour >= 10 || digit) {
				// Either the tens digit is significant, or we need a leading
				// zero so that a trailing digit will not be read as part of the
				// the colour code.
				*--p = '0' + (rhs.colour / 10);
			}
			*--p = Colour;
		}
		else if (!digit) {
			// Revert to default colour. There is no trailing digit that might
			// be read as part of a colour code.
			*--p = Colour;
		}
	}

	return p;
}

This code doesn't support background colours, but the same approach works in that case too.