x86_modrm.c

// 16-bit and 32-bit modrm encoder in x86.
// nothing fancy

FILE *out;

// The bitness of the output
// within operating system code can change from one
// instruction to another instruction.
int bits;

// Register numbers as in the CPU machine code
enum cpu_regs {
    cpu_ax=0,cpu_cx,cpu_dx,cpu_bx,cpu_sp,cpu_bp,cpu_si,cpu_di,
    cpu_al=0,cpu_cl,cpu_dl,cpu_bl,cpu_ah,cpu_ch,cpu_dh,cpu_bh,
    cpu_eax=0,cpu_ecx,cpu_edx,cpu_ebx,cpu_esp,cpu_ebp,cpu_esi,cpu_edi,
    cpu_cs=0,cpu_ss,cpu_ds,cpu_es,cpu_fs,cpu_gs,
};

// Segment override prefixes
int seg_overrides[] = {
    [cpu_cs] = 0x2e,
    [cpu_ss] = 0x36,
    [cpu_ds] = 0x3e,
    [cpu_es] = 0x26,
    [cpu_fs] = 0x64,
    [cpu_gs] = 0x65,
};

// output byte
static void obyte(int byte)
{
    fputc(byte, out);
}

static int omodrm(int mod, int reg, int rm)
{
    int byte = 0;
    byte |= mod << 6;
    byte |= reg << 3;
    byte |= rm;
    obyte(byte);
}

static int osib(int ss, int iii, int bbb)
{
    int byte = 0;
    byte |= ss << 6;
    byte |= iii << 3;
    byte |= bbb;
    obyte(byte);
}

// In 16 bit addressing mode [base+index+disp], only 
// the following combinations are allowed:
//   [disp]
//   [si/di/bx/bp]
//   [si/di/bx/bp + disp]
//   [bx/bp + si/di]
//   [bx/bp + si/di + disp]
// In such case scale is expected to be 1. In 32-bit mode,
//   esp is not allowed to be indexed. Everything else is probably
//   allowed. Well, in any case additional verification needs to be
//   done on a higher level, such as index can't be 3 or 5 and stuff.
static bool is_modrm_ok(int amode, int seg, int base, int index, int scale, uint disp)
{
    if(amode == 16) {
        if(scale != 1) return false;

        if(base == -1 && !(index == -1)) return false;
        if((base == cpu_si || base == cpu_di) && !(index == -1)) return false;
        if(base == cpu_bx || base == cpu_bp && !(index == -1 || index == cpu_si || index == cpu_di)) return false;

        if(disp > 0xffff) return false;
        return true;
    }
    else if(amode == 32) {
        if(index == cpu_esp) return false;
    }
    return true;
}

// modrm for memory operands for 16-bit mode.
static inline void mem_modrm16(int base, int index, int scale, uint disp, int reg)
{
    assert(scale == 1);
    assert(disp <= 0xffff);

    bool force_disp = false;
    int rm = -1;
    
    if(index == -1) {
        if (base == cpu_si) rm = 4;
        else if (base == cpu_di) rm = 5;
        else if (base == -1) {rm = 6; force_disp = true;}
        else if (base == cpu_bx) rm = 7;
    }
    else if(base == cpu_bx) {
        if      (index == cpu_si) rm = 0;
        else if (index == cpu_di) rm = 1;
    }
    else if(base == cpu_bp) {
        if      (index == cpu_si) rm = 2;
        else if (index == cpu_di) rm = 3;
    }

    int mod;
    if      (disp == 0 && !force_disp) mod = 0;
    else if (disp <= 0xff)             mod = 1;
    else if (disp <= 0xffff)           mod = 2;

    omodrm(mod, reg, rm);

    if(mod==1||mod==2) obyte(disp & 0xff);
    if(mod==2)         obyte((disp>>8) & 0xff);
}

// modrm for 32-bit memory operands
static inline void mem_modrm32(int base, int index, int scale, uint disp, int reg) {
    int mm;
    int rrr;
    int ss;
    int iii;
    int bbb;

    if(index == -1) {
        // [disp]
        if (base == -1) {
            rrr = 5;
            mm = 0;
        }
        // [reg]
        // [reg+disp]
        else {
            rrr = base;

            if(base != cpu_ebp && disp == 0) mm = 0;
            else if(disp < 0xff) mm=1;
            else mm=2;
        }
    }
    else {
        rrr = 4;

        if(scale == 1) ss = 0;
        if(scale == 2) ss = 1;
        if(scale == 4) ss = 2;
        if(scale == 8) ss = 3;

        iii = index;

        // [scale*index+disp]
        if(base == -1) {
            mm = 0;
            bbb = 5;
        }
        else {
            if(base != cpu_ebp && disp == 0) mm = 0;
            else if(disp < 0xff) mm=1;
            else mm=2;
            bbb = base;
        }

    }

    // output
    omodrm(mm, reg, rrr);
    if(rrr == 4) {
        osib(ss, iii, bbb);
    }

    int dispsize = 0;
    if(mm == 0 && (rrr == 5 || (rrr == 4 && bbb == 5))) {
        dispsize = 32;
    }
    else if(mm == 1) {
        dispsize = 8;
    }
    else if(mm == 2) {
        dispsize = 32;
    }

    if(dispsize == 8) {
        obyte(disp);
    }
    else if(dispsize == 32) {
        obyte((disp >>  0)&0xff);
        obyte((disp >>  8)&0xff);
        obyte((disp >> 16)&0xff);
        obyte((disp >> 24)&0xff);
    }
}


// generic memory operand modrm
static void mem_modrm(int asize, int base, int index, int scale, uint disp, int reg) {
    if(asize == 32) mem_modrm32(base,index,scale,disp,reg);
    if(asize == 16) mem_modrm16(base,index,scale,disp,reg);
}



// Operand abstraction
enum operand_kind typedef operand_kind;
enum operand_kind {
    operand_number,
    operand_gpreg,
    operand_sreg,
    operand_mem,
};

struct operand typedef operand;
struct operand {
    operand_kind kind;
    uint number;
    int amode;
    int seg;
    int reg;
    int index;
    int scale;
};



// Example: mov8 and mov16 instruction implementation

static void mov8(operand to, operand from)
{
    if(to.kind == operand_mem) {
        if(to.seg != cpu_ds) obyte(seg_overrides[to.seg]);
        obyte(0x88);
        mem_modrm(to.amode, to.reg, to.index, to.scale, to.number, from.reg);
    }
    else if (from.kind == operand_mem){
        if(to.seg != cpu_ds) obyte(seg_overrides[from.seg]);
        obyte(0x88);
        mem_modrm(from.amode, from.reg, from.index, from.scale, from.number, to.reg);
    }
    else {
        obyte(0x88);
        omodrm(3, from.reg, to.reg);
    }
}

static void mov16(operand to, operand from)
{
    if(bits != 16) obyte(0x66);
    if(to.kind == operand_mem) {
        if(bits != 16) obyte(0x67);
        if(to.seg != cpu_ds) obyte(seg_overrides[to.seg]);
        obyte(0x89);
        mem_modrm(to.amode, to.reg, to.index, to.scale, to.number, from.reg);
    }
    else if(from.kind == operand_mem) {
        if(bits != 16) obyte(0x67);
        if(from.seg != cpu_ds) obyte(seg_overrides[to.seg]);
        obyte(0x89);
        mem_modrm(from.amode, from.reg, from.index, from.scale, from.number, to.reg);
    }
    else {
        obyte(0x89);
        omodrm(3, from.reg, to.reg);
    }
}