x64 machine code emitter

x64_emitter.cpp

#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <windows.h>
#include <stdint.h>

#define Assert(x) \
    if (!(x)) { MessageBoxA(0, #x, "Assertion Failure", MB_OK); __debugbreak(); }

enum Register {
    RAX = 0,
    RCX = 1,
    RDX = 2,
    RBX = 3,
    RSP = 4,
    RBP = 5,
    RSI = 6,
    RDI = 7,
    R8 = 8,
    R9 = 9,
    R10 = 10,
    R11 = 11,
    R12 = 12,
    R13 = 13,
    R14 = 14,
    R15 = 15
};

enum Scale {
    X1 = 0,
    X2 = 1,
    X4 = 2,
    X8 = 3
};

enum Mode {
    INDIRECT = 0,
    INDIRECT_BYTE_DISPLACED = 1,
    INDIRECT_DISPLACED = 2,
    DIRECT = 3,
};

enum ConditionCode {
    O,
    NO,
    B,
    NB,
    E,
    NE,
    NA,
    A,
    S,
    NS,
    P,
    NP,
    L,
    NL,
    NG,
    G,
    NAE = B,
    C = B,
    AE = NB,
    NC = NB,
    Z = E,
    NZ = NE,
    BE = NA,
    NBE = A,
    PE = P,
    PO = NP,
    NGE = L,
    GE = NL,
    LE = NG,
    NLE = G
};

uint8_t code[1024 * 1024];
uint8_t *next_code = code;

void Emit(uint8_t value) {
    *next_code = value;
    next_code++;
}

void Emit4(uint32_t value) {
    Emit(value & 0xFF);
    Emit((value >> 8) & 0xFF);
    Emit((value >> 16) & 0xFF);
    Emit((value >> 24) & 0xFF);
}

void Emit8(uint64_t value) {
    Emit(value & 0xFF);
    Emit((value >> 8) & 0xFF);
    Emit((value >> 16) & 0xFF);
    Emit((value >> 24) & 0xFF);
    Emit((value >> 32) & 0xFF);
    Emit((value >> 40) & 0xFF);
    Emit((value >> 48) & 0xFF);
    Emit((value >> 56) & 0xFF);
}

void EmitModRxRm(uint8_t mod, uint8_t rx, uint8_t rm) {
    Assert(mod < 4);
    Assert(rx < 16);
    Assert(rm < 16);
    Emit((mod << 6) | ((rx & 7) << 3) | (rm & 7));
}

void EmitRexIndexed(uint8_t rx, uint8_t base, uint8_t index) {
    Emit(0x48 | (base >> 3) | ((index >> 3) << 1) | ((rx >> 3) << 2));
}

void EmitRex(uint8_t rx, uint8_t base) {
    EmitRexIndexed(rx, base, 0);
}

// op rax, rcx
// rx = RAX, reg = RCX
void EmitDirect(uint8_t rx, Register reg) {
    EmitModRxRm(DIRECT, rx, reg);
}

// op rax, [rcx]
// rx = RAX, base = RCX
void EmitIndirect(uint8_t rx, Register base) {
    Assert((base & 7) != RSP);
    Assert((base & 7) != RBP);
    EmitModRxRm(INDIRECT, rx, base);
}

// op rax, [rip + 0x12345678]
// rx = RAX, displacement = 0x12345678
void EmitIndirectDisplacedRip(uint8_t rx, int32_t displacement) {
    EmitModRxRm(INDIRECT, rx, RBP);
    Emit4(displacement);
}

// op rax, [rcx + 0x12]
// rx = RAX, base = RCX, displacement = 0x12
void EmitIndirectByteDisplaced(uint8_t rx, Register base, int8_t displacement) {
    Assert((base & 7) != RSP);
    EmitModRxRm(INDIRECT_BYTE_DISPLACED, rx, base);
    Emit(displacement);
}

// op rax, [rcx + 0x12345678]
// rx = RAX, base = RCX, displacement = 0x12345678
void EmitIndirectDisplaced(uint8_t rx, Register base, int32_t displacement) {
    Assert((base & 7) != RSP);
    EmitModRxRm(INDIRECT_DISPLACED, rx, base);
    Emit4(displacement);
}

// op rax, [rcx + 4*rdx]
// rx = RAX, base = RCX, index = RDX, scale = X4
void EmitIndirectIndexed(uint8_t rx, Register base, Register index, Scale scale) {
    Assert((base & 7) != RBP);
    EmitModRxRm(INDIRECT, rx, RSP);
    EmitModRxRm(scale, index, base);
}

// op rax, [rcx + 4*rdx + 0x12]
// rx = RAX, base = RCX, index = RDX, scale = X4, displacement = 0x12
void EmitIndirectIndexedByteDisplaced(uint8_t rx, Register base, Register index, Scale scale, int8_t displacement) {
    EmitModRxRm(INDIRECT_BYTE_DISPLACED, rx, RSP);
    EmitModRxRm(scale, index, base);
    Emit(displacement);
}

// op rax, [rcx + 4*rdx + 0x12345678]
// rx = RAX, base = RCX, index = RDX, scale = X4, displacement = 0x12345678
void EmitIndirectIndexedDisplaced(uint8_t rx, Register base, Register index, Scale scale, int32_t displacement) {
    EmitModRxRm(INDIRECT_DISPLACED, rx, RSP);
    EmitModRxRm(scale, index, base);
    Emit4(displacement);
}

// op rax, [0x12345678]
// rx = RAX, displacement = 0x12345678
void EmitDisplaced(uint8_t rx, int32_t displacement) {
    EmitModRxRm(INDIRECT, rx, RSP);
    EmitModRxRm(X1, RSP, RBP);
    Emit4(displacement);
}

#define EMIT_I(operation, source_immediate) \
    EmitRex(0, 0); \
    Emit_##operation##_I(); \
    Emit4(source_immediate)

#define EMIT_MOV_R_I(destination, source_immediate) \
    EmitRex(destination, 0); \
    Emit(0xB8); \
    Emit8(source_immediate)

#define EMIT_MOV_RAX_OFF(source_offset) \
    EmitRex(0, 0); \
    Emit(0xA1); \
    Emit8(source_offset)

#define EMIT_MOV_OFF_RAX(destination_offset) \
    EmitRex(0, 0); \
    Emit(0xA3); \
    Emit8(destination_offset)

#define EMIT_R_R(operation, destination, source) \
    EmitRex(destination, source); \
    Emit_##operation##_R(); \
    EmitDirect(destination, source)

#define EMIT_R_RIPD(operation, destination, source_displacement) \
    EmitRex(destination, 0); \
    Emit_##operation##_R(); \
    EmitIndirectDisplacedRip(destination, source_displacement);

#define EMIT_R_D(operation, destination, source_displacement) \
    EmitRex(destination, 0); \
    Emit_##operation##_R(); \
    EmitDisplaced(destination, source_displacement)

#define EMIT_R_M(operation, destination, source) \
    EmitRex(destination, source); \
    Emit_##operation##_R(); \
    EmitIndirect(destination, source)

#define EMIT_R_MD1(operation, destination, source, source_displacement) \
    EmitRex(destination, source); \
    Emit_##operation##_R(); \
    EmitIndirectByteDisplaced(destination, source, source_displacement)

#define EMIT_R_MD(operation, destination, source, source_displacement) \
    EmitRex(destination, source); \
    Emit_##operation##_R(); \
    EmitIndirectDisplaced(destination, source, source_displacement)

#define EMIT_R_SIB(operation, destination, source_base, source_scale, source_index) \
    EmitRexIndexed(destination, source_base, source_index); \
    Emit_##operation##_R(); \
    EmitIndirectIndexed(destination, source_base, source_index, source_scale)

#define EMIT_R_SIBD1(operation, destination, source_base, source_scale, source_index, source_displacement) \
    EmitRexIndexed(destination, source_base, source_index); \
    Emit_##operation##_R(); \
    EmitIndirectIndexedByteDisplaced(destination, source_base, source_index, source_scale, source_displacement)

#define EMIT_R_SIBD(operation, destination, source_base, source_scale, source_index, source_displacement) \
    EmitRexIndexed(destination, source_base, source_index); \
    Emit_##operation##_R(); \
    EmitIndirectIndexedDisplaced(destination, source_base, source_index, source_scale, source_displacement)

#define EMIT_M_R(operation, destination, source) \
    EmitRex(source, destination); \
    Emit_##operation##_M(); \
    EmitIndirect(source, destination)

#define EMIT_D_R(operation, destination_displacement, source) \
    EmitRex(source, 0); \
    Emit_##operation##_M(); \
    EmitDisplaced(source, destination_displacement)

#define EMIT_RIPD_R(operation, destination_displacement, source) \
    EmitRex(source, 0); \
    Emit_##operation##_M(); \
    EmitIndirectDisplacedRip(source, destination_displacement);

#define EMIT_MD1_R(operation, destination, destination_displacement, source) \
    EmitRex(source, destination); \
    Emit_##operation##_M(); \
    EmitIndirectByteDisplaced(source, destination, destination_displacement)

#define EMIT_MD_R(operation, destination, destination_displacement, source) \
    EmitRex(source, destination); \
    Emit_##operation##_M(); \
    EmitIndirectDisplaced(source, destination, destination_displacement)

#define EMIT_SIB_R(operation, destination_base, destination_scale, destination_index, source) \
    EmitRexIndexed(source, destination_base, destination_index); \
    Emit_##operation##_M(); \
    EmitIndirectIndexed(source, destination_base, destination_index, destination_scale)

#define EMIT_SIBD1_R(operation, destination_base, destination_scale, destination_index, destination_displacement, source) \
    EmitRexIndexed(source, destination_base, destination_index); \
    Emit_##operation##_M(); \
    EmitIndirectIndexedByteDisplaced(source, destination_base, destination_index, destination_scale, destination_displacement)

#define EMIT_SIBD_R(operation, destination_base, destination_scale, destination_index, destination_displacement, source) \
    EmitRexIndexed(source, destination_base, destination_index); \
    Emit_##operation##_M(); \
    EmitIndirectIndexedDisplaced(source, destination_base, destination_index, destination_scale, destination_displacement)

#define EMIT_R_I(operation, destination, source_immediate) \
    EmitRex(0, destination); \
    Emit_##operation##_I(); \
    EmitDirect(extension_##operation##_I, destination); \
    Emit4(source_immediate)

#define EMIT_M_I(operation, destination, source_immediate) \
    EmitRex(0, destination); \
    Emit_##operation##_I(); \
    EmitIndirect(extension_##operation##_I, destination); \
    Emit4(source_immediate)

#define EMIT_D_I(operation, destination_displacement, source_immediate) \
    EmitRex(0, 0); \
    Emit_##operation##_I(); \
    EmitDisplaced(extension_##operation##_I, destination_displacement); \
    Emit4(source_immediate)

#define EMIT_RIPD_I(operation, destination_displacement, source_immediate) \
    EmitRex(0, 0); \
    Emit_##operation##_I(); \
    EmitIndirectDisplacedRip(extension_##operation##_I, destination_displacement); \
    Emit4(source_immediate)

#define EMIT_MD1_I(operation, destination, destination_displacement, source_immediate) \
    EmitRex(0, destination); \
    Emit_##operation##_I(); \
    EmitIndirectByteDisplaced(extension_##operation##_I, destination, destination_displacement); \
    Emit4(source_immediate)

#define EMIT_MD_I(operation, destination, destination_displacement, source_immediate) \
    EmitRex(0, destination); \
    Emit_##operation##_I(); \
    EmitIndirectDisplaced(extension_##operation##_I, destination, destination_displacement); \
    Emit4(source_immediate)

#define EMIT_SIB_I(operation, destination_base, destination_scale, destination_index, source_immediate) \
    EmitRexIndexed(0, destination_base, destination_index); \
    Emit_##operation##_I(); \
    EmitIndirectIndexed(extension_##operation##_I, destination_base, destination_index, destination_scale); \
    Emit4(source_immediate)

#define EMIT_SIBD1_I(operation, destination_base, destination_scale, destination_index, destination_displacement, source_immediate) \
    EmitRexIndexed(0, destination_base, destination_index); \
    Emit_##operation##_I(); \
    EmitIndirectIndexedByteDisplaced(extension_##operation##_I, destination_base, destination_index, destination_scale, destination_displacement); \
    Emit4(source_immediate)

#define EMIT_SIBD_I(operation, destination_base, destination_scale, destination_index, destination_displacement, source_immediate) \
    EmitRexIndexed(0, destination_base, destination_index); \
    Emit_##operation##_I(); \
    EmitIndirectIndexedDisplaced(extension_##operation##_I, destination_base, destination_index, destination_scale, destination_displacement); \
    Emit4(source_immediate)

#define EMIT_X_R(operation, source) \
    EmitRex(0, source); \
    Emit_##operation##_X(); \
    EmitDirect(extension_##operation##_X, source)

#define EMIT_X_RIPD(operation, source_displacement) \
    EmitRex(0, 0); \
    Emit_##operation##_X(); \
    EmitIndirectDisplacedRip(extension_##operation##_X, source_displacement);

#define EMIT_X_D(operation, source_displacement) \
    EmitRex(0, 0); \
    Emit_##operation##_X(); \
    EmitDisplaced(extension_##operation##_X, source_displacement)

#define EMIT_X_M(operation, source) \
    EmitRex(0, source); \
    Emit_##operation##_X(); \
    EmitIndirect(extension_##operation##_X, source)

#define EMIT_X_MD1(operation, source, source_displacement) \
    EmitRex(0, source); \
    Emit_##operation##_X(); \
    EmitIndirectByteDisplaced(extension_##operation##_X, source, source_displacement)

#define EMIT_X_MD(operation, source, source_displacement) \
    EmitRex(0, source); \
    Emit_##operation##_X(); \
    EmitIndirectDisplaced(extension_##operation##_X, source, source_displacement)

#define EMIT_X_SIB(operation, source_base, source_scale, source_index) \
    EmitRexIndexed(0, source_base, source_index); \
    Emit_##operation##_X(); \
    EmitIndirectIndexed(extension_##operation##_X, source_base, source_index, source_scale)

#define EMIT_X_SIBD1(operation, source_base, source_scale, source_index, source_displacement) \
    EmitRexIndexed(0, source_base, source_index); \
    Emit_##operation##_X(); \
    EmitIndirectIndexedByteDisplaced(extension_##operation##_X, source_base, source_index, source_scale, source_displacement)

#define EMIT_X_SIBD(operation, source_base, source_scale, source_index, source_displacement) \
    EmitRexIndexed(0, source_base, source_index); \
    Emit_##operation##_X(); \
    EmitIndirectIndexedDisplaced(extension_##operation##_X, source_base, source_index, source_scale, source_displacement)

#define EMIT_C_I(operation, condition_code, source_immediate) \
    Emit_##operation##_C_I(condition_code); \
    Emit4(source_immediate)

#define OP1R(operation, opcode) \
    void Emit_##operation##_R() { \
        Emit(opcode); \
    }

#define OP1M(operation, opcode) \
    void Emit_##operation##_M() { \
        Emit(opcode); \
    }

#define OP1I(operation, opcode, extension) \
    void Emit_##operation##_I() { \
        Emit(opcode); \
    } \
    enum { extension_##operation##_I = extension };

#define OP1X(operation, opcode, extension) \
    void Emit_##operation##_X() { \
        Emit(opcode); \
    } \
    enum { extension_##operation##_X = extension };

#define OP2CI(operation, opcode) \
    void Emit_##operation##_C_I(ConditionCode condition_code) { \
        Emit(0x0F); \
        Emit(opcode + condition_code); \
    }

// Instructions

OP1R(MOV, 0x8B)
OP1M(MOV, 0x89)

OP1R(ADD, 0x03)
OP1M(ADD, 0x01)
OP1I(ADD, 0x81, 0x00)

OP1R(AND, 0x23)
OP1M(AND, 0x21)
OP1I(AND, 0x81, 0x04)

OP1X(MUL, 0xF7, 0x04)

OP1I(JMP, 0xE9, 0x00)

OP2CI(J, 0x80)

// Tests

extern "C" void assembly_test();

int main(int argc, char **argv) {
    EMIT_MOV_R_I(RBX, 0x12345678deadbeefull);
    EMIT_MOV_RAX_OFF(0x12345678deadbeefull);
    EMIT_MOV_OFF_RAX(0x12345678deadbeefull);
    EMIT_R_R(MOV, RAX, R10);
    EMIT_M_R(MOV, RAX, R10);
    EMIT_I(JMP, 0x1234);
    EMIT_C_I(J, NZ, 0x1234);
    EMIT_C_I(J, NZ, 0x1234);
    EMIT_C_I(J, NZ, 0x1234);
    EMIT_D_I(ADD, 0x1234578, 0xDEADBEEF);
    EMIT_RIPD_I(ADD, 0x1234578, 0xDEADBEEF);
    for (uint8_t d = RAX; d <= R15; d++) {
        Register destination = (Register)d;
        EMIT_X_R(MUL, destination);
        if ((destination & 7) != RSP) {
            EMIT_X_MD1(MUL, destination, 0x12);
            EMIT_X_MD(MUL, destination, 0x12345678);
            if ((destination & 7) != RBP) {
                EMIT_X_M(MUL, destination);
                EMIT_X_SIB(MUL, destination, X4, R8);
                EMIT_X_SIBD1(MUL, destination, X4, R8, 0x12);
                EMIT_X_SIBD(MUL, destination, X4, R8, 0x12345678);
            }
        }
        EMIT_R_I(ADD, destination, 0xDEADBEEF);
        if ((destination & 7) != RSP) {
            EMIT_MD1_I(ADD, destination, 0x12, 0xDEADBEEF);
            EMIT_MD_I(ADD, destination, 0x12345678, 0xDEAFBEEF);
            if ((destination & 7) != RBP) {
                EMIT_SIB_I(ADD, destination, X4, R8, 0xDEADBEEF);
            }
        }
        EMIT_R_RIPD(ADD, destination, 0x1235678);
        EMIT_R_D(ADD, destination, 0x1235678);
        EMIT_RIPD_R(ADD, 0x1235678, destination);
        EMIT_D_R(ADD, 0x1235678, destination);
        for (uint8_t s = RAX; s <= R15; s++) {
            Register source = (Register)s;
            EMIT_R_R(ADD, destination, source);
            if ((source & 7) != RBP) {
                EMIT_R_SIB(ADD, destination, source, X4, destination);
                EMIT_R_SIBD1(ADD, destination, source, X4, destination, 0x12);
                EMIT_R_SIBD(ADD, destination, source, X4, destination, 0x12345678);
            }
            if ((destination & 7) != RBP) {
                EMIT_SIB_R(ADD, destination, X4, source, source);
                EMIT_SIBD1_R(ADD, destination, X4, source, 0x12, source);
                EMIT_SIBD_R(ADD, destination, X4, source, 0x12345678, source);
            }
            if ((source & 7) != RSP && (source & 7) != RBP) {
                EMIT_R_M(ADD, destination, source);
                EMIT_R_MD1(ADD, destination, source, 0x12);
                EMIT_R_MD(ADD, destination, source, 0x12345678);
            }
            if ((destination & 7) != RSP && (destination & 7) != RBP) {
                EMIT_M_R(ADD, destination, source);
                EMIT_MD1_R(ADD, destination, 0x12, source);
                EMIT_MD_R(ADD, destination, 0x12345678, source);
            }
            if ((source & 7) == RSP) {
                EMIT_R_SIB(ADD, destination, source, X1, RSP);
                #if 0
                EmitRex(destination, source);
                EmitAddToRegister();
                EmitIndirectIndexed(destination, source, RSP, X1);
                #endif
            }
        }
    }
    FILE *file = fopen("d:\\be\\test.bin", "wb");
    fwrite(code, next_code - code, 1, file);
    fclose(file);
//	assembly_test();
    return 0;
}