x64_disassembler.c

struct x64_mov {
  int length;
  int is_avx;
  int is_sse;
  int is_load;
  int is_indirect;
  int has_imm;
  int has_base;
  int has_index;
  int operand_size;
  int reg;
  int base;
  int index;
  int scale;
  int disp;
  uint64_t imm;
};

/* working buffer to aid in disassembling the more complex AVX instructions
   which have special bits in their VEX prefix that effectively push other
   bytes into the decode stream */
#define X64_MAX_INSTR_LEN 15

struct x64_decbuf {
  const uint8_t *data;
  uint8_t tmp[X64_MAX_INSTR_LEN];
  int head;
  int tail;
};

static void pref(struct x64_decbuf *dec, int n) {
  while (dec->tail < (dec->head + n)) {
    CHECK_LT(dec->tail, X64_MAX_INSTR_LEN);
    dec->tmp[dec->tail++] = *(dec->data++);
  }
}

static void push(struct x64_decbuf *dec, uint8_t c) {
  CHECK_LT(dec->tail, X64_MAX_INSTR_LEN);
  dec->tmp[dec->tail++] = c;
}

static void skip(struct x64_decbuf *dec, int n) {
  dec->head += n;
}

static uint8_t peek(struct x64_decbuf *dec) {
  pref(dec, 1);

  return dec->tmp[dec->head];
}

static uint16_t peek16(struct x64_decbuf *dec) {
  pref(dec, 2);

  return *(uint16_t *)&dec->tmp[dec->head];
}

static uint8_t pop(struct x64_decbuf *dec) {
  pref(dec, 1);

  uint8_t d = dec->tmp[dec->head];
  dec->head += 1;
  return d;
}

static uint16_t pop16(struct x64_decbuf *dec) {
  pref(dec, 2);

  uint16_t d = *(uint16_t *)&dec->tmp[dec->head];
  dec->head += 2;
  return d;
}

static uint32_t pop32(struct x64_decbuf *dec) {
  pref(dec, 4);

  uint32_t d = *(uint32_t *)&dec->tmp[dec->head];
  dec->head += 4;
  return d;
}

static uint64_t pop64(struct x64_decbuf *dec) {
  pref(dec, 8);

  uint64_t d = *(uint64_t *)&dec->tmp[dec->head];
  dec->head += 8;
  return d;
}

#define PUSH(c) push(&dec, c)
#define SKIP(n) skip(&dec, n)
#define PEEK() peek(&dec)
#define PEEK16() peek16(&dec)
#define POP() pop(&dec)
#define POP16() pop16(&dec)
#define POP32() pop32(&dec)
#define POP64() pop64(&dec)

int x64_decode_mov(const uint8_t *data, struct x64_mov *mov) {
  struct x64_decbuf dec = {data, {}, 0, 0};

  /* modifiers specified by VEX or REX prefixes */
  int w = 0;
  int r = 0;
  int x = 0;
  int b = 0;

  /* test for VEX prefix */
  int is_avx = 0;

  if (PEEK() == 0xc4) {
    int p = 0;
    int m = 0;

    is_avx = 1;
    SKIP(1);

    uint8_t vex = POP();
    r = ~vex & 0b10000000;
    x = ~vex & 0b01000000;
    b = ~vex & 0b00100000;
    m = vex & 0b00011111;

    vex = POP();
    w = vex & 0b10000000;
    /*v = vex & 0b01111000;*/
    /*l = vex & 0b00000100;*/
    p = vex & 0b00000011;

    /* push additional prefix bytes */
    switch (p) {
      case 1:
        PUSH(0x66);
        break;
      case 2:
        PUSH(0xf3);
        break;
      case 3:
        PUSH(0xf2);
        break;
    }

    /* push additional opcode bytes */
    switch (m) {
      case 1:
        PUSH(0x0f);
        break;
      case 2:
        PUSH(0x0f);
        PUSH(0x38);
        break;
      case 3:
        PUSH(0x0f);
        PUSH(0x3a);
        break;
    }
  }

  /* test for SSE prefix */
  int is_sse = 0;
  int is_double = 0;

  if (PEEK() == 0xf2 || PEEK() == 0xf3) {
    uint8_t b = POP();
    is_sse = 1;
    is_double = b == 0xf2;
  }

  /* test for operand size prefix */
  int has_opprefix = 0;

  if (PEEK() == 0x66) {
    uint8_t b = POP();
    has_opprefix = 1;
  }

  /* test for REX prefix
     http://wiki.osdev.org/X86-64_Instruction_Encoding#Encoding */
  if ((PEEK() & 0xf0) == 0x40) {
    uint8_t rex = POP();
    w = rex & 0b1000;
    r = rex & 0b0100;
    x = rex & 0b0010;
    b = rex & 0b0001;
  }

  /* test for MOV opcode
     http://x86.renejeschke.de/html/file_module_x86_id_176.html */
  int is_load = 0;
  int has_imm = 0;
  int operand_size = 0;

  /* MOV r8,r/m8
     MOV r16,r/m16
     MOV r32,r/m32
     MOV r64,r/m64 */
  if (PEEK() == 0x8a || PEEK() == 0x8b) {
    uint8_t b = POP();
    is_load = 1;
    has_imm = 0;
    operand_size = b == 0x8a ? 1 : (has_opprefix ? 2 : (w ? 8 : 4));
  }
  /* MOV r/m8,r8
     MOV r/m16,r16
     MOV r/m32,r32
     MOV r/m64,r64 */
  else if (PEEK() == 0x88 || PEEK() == 0x89) {
    uint8_t b = POP();
    is_load = 0;
    has_imm = 0;
    operand_size = b == 0x88 ? 1 : (has_opprefix ? 2 : (w ? 8 : 4));
  }
  /* MOV r8,imm8
     MOV r16,imm16
     MOV r32,imm32 */
  else if (PEEK() == 0xb0 || PEEK() == 0xb8) {
    uint8_t b = POP();
    is_load = 1;
    has_imm = 1;
    operand_size = b == 0xb0 ? 1 : (has_opprefix ? 2 : 4);
  }
  /* MOV r/m8,imm8
     MOV r/m16,imm16
     MOV r/m32,imm32 */
  else if (PEEK() == 0xc6 || PEEK() == 0xc7) {
    uint8_t b = POP();
    is_load = 0;
    has_imm = 1;
    operand_size = b == 0xc6 ? 1 : (has_opprefix ? 2 : 4);
  }
  /* MOVSS xmm1, xmm2/m32
     MOVSD xmm1, xmm2/m64 */
  else if (is_sse && PEEK16() == 0x100f) {
    is_load = 1;
    operand_size = is_double ? 8 : 4;
    SKIP(2);
  }
  /* MOVSS xmm2/m32, xmm1
     MOVSD xmm2/m64, xmm1 */
  else if (is_sse && PEEK16() == 0x110f) {
    is_load = 0;
    operand_size = is_double ? 8 : 4;
    SKIP(2);
  }
  /* not a supported MOV instruction */
  else {
    return 0;
  }

  /* process ModR/M byte */
  uint8_t modrm = POP();
  uint8_t modrm_mod = (modrm & 0b11000000) >> 6;
  uint8_t modrm_reg = (modrm & 0b00111000) >> 3;
  uint8_t modrm_rm = (modrm & 0b00000111);

  mov->is_avx = is_avx;
  mov->is_sse = is_sse;
  mov->is_load = is_load;
  mov->is_indirect = (modrm_mod != 0b11);
  mov->has_imm = has_imm;
  mov->has_base = 0;
  mov->has_index = 0;
  mov->operand_size = operand_size;
  mov->reg = modrm_reg + (r ? 8 : 0);
  mov->base = 0;
  mov->index = 0;
  mov->scale = 0;
  mov->disp = 0;
  mov->imm = 0;

  /* process optional SIB byte */
  if (modrm_rm == 0b100) {
    uint8_t sib = POP();
    uint8_t sib_scale = (sib & 0b11000000) >> 6;
    uint8_t sib_index = (sib & 0b00111000) >> 3;
    uint8_t sib_base = (sib & 0b00000111);

    mov->has_base = (modrm_mod != 0b00 || sib_base != 0b101);
    mov->has_index = (sib_index != 0b100);
    mov->base = sib_base + (b ? 8 : 0);
    mov->index = sib_index + (x ? 8 : 0);
    mov->scale = sib_scale;
  } else {
    mov->has_base = 1;
    mov->base = modrm_rm + (b ? 8 : 0);
  }

  /* process optional displacement */
  switch (modrm_mod) {
    case 0b00: {
      /* RIP-relative */
      if (modrm_rm == 0b101) {
        mov->disp = POP32();
      }
    } break;

    case 0b01: {
      mov->disp = POP();
    } break;

    case 0b10: {
      mov->disp = POP32();
    } break;
  }

  /* process optional immediate */
  if (mov->has_imm) {
    switch (mov->operand_size) {
      case 1: {
        mov->imm = POP();
      } break;

      case 2: {
        mov->imm = POP16();
      } break;

      case 4: {
        mov->imm = POP32();
      } break;

      case 8: {
        mov->imm = POP64();
      } break;
    }
  }

  /* calculate total instruction length */
  mov->length = (int)(dec.data - data);

  return 1;
}