Simple x86 (GAS/AT&T flavor asm) emulator for Riko4

0_x86emu.lua

local cursor = require("ui.cursor").new()

local running = true
local env = {
  rax = 0,
  rbx = 0,
  rcx = 0,
  rdx = 0,
  rsi = 0,
  rdi = 0,
  rbp = 0,
  rsp = 0x1000,
  r8 = 0,
  r9 = 0,
  r10 = 0,
  r11 = 0,
  r12 = 0,
  r13 = 0,
  r14 = 0,
  r15 = 0,

  rip = 0,

  zf = 0, sf = 0
}

local order = {
  "rax","rbx","rcx","rdx","rsi","rdi","rbp","rsp","r8",
  "r9","r10","r11","r12","r13","r14","r15","rip", "zf", "sf"
}

local pageSize = 4096
local pages = {
  {
    name = "stack",
    startVirtual = 0x800,
    physical = ffi.new("uint8_t[" .. pageSize .. "]")
  }
}

local function findPage(address)
  for i = 1, #pages do
    local page = pages[i]
    if address >= page.startVirtual and address <= page.startVirtual + pageSize then
      return page.physical, address - page.startVirtual
    end
  end

  error(("{%d} Segfault (page fault): 0x%x"):format(tonumber(env.rip), tonumber(address)), 0)
end

local function readBytes(address, size)
  local page, index = findPage(address)
  local box = 0
  if size > 4 then
    box = ffi.new("uint64_t")
  end

  for i = size, 1, -1 do
    box = bit.lshift(box, 8) + page[index + i - 1]
  end

  return box
end

local function writeBytes(address, size, value)
  local page, index = findPage(address)

  for i = 1, size do
    page[index + i - 1] = ffi.cast("uint8_t", value)
    value = bit.rshift(value, 8)
  end
end

-- TODO: Remove stack, it is an unnecessary abstraction
local stack = setmetatable({}, {
  __index = function(t, k)
    return readBytes(k, 8)
  end,
  __newindex = function(t, k, v)
    writeBytes(k, 8, v)
  end
})

local args = {...}
local filename = args[1]

local fileData
do
  local fileHandle = fs.open(filename, "r")
  fileData = fileHandle:read("*a")
  fileHandle:close()
end

local lines = {}
for line in fileData:gmatch("[^\n]+") do
  lines[#lines + 1] = line:match("^[^#]+")
end

local labels = {}
for i = 1, #lines do
  local line = lines[i]
  local label = line:match("^(.?%w+):")
  if label then
    labels[label] = i
  end
end

local labelStack
if labels.main then
  env.rip = labels.main
  labelStack = {"main"}
elseif args[2] then
  env.rip = labels[args[2]]
  labelStack = {args[2]}
  if not env.rip then
    print(("No label '%s' found, aborting.."):format(args[2]))
    return
  end
else
  print("No main function found, and no label specified, aborting..")
  return
end

local hexlookup = {[0] = "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"}
local function tohexstr(num, min)
  min = min or 1
  local str, i = "", 0
  while num > 0 or i < min do
    local lsb = bit.band(num, 0xF)
    str = hexlookup[tonumber(lsb)] .. str
    num = bit.rshift(num, 4)
    i = i + 1
  end

  return "0x"..str
end

local function trim(str)
  local front = str:match("^%s*(.+)") or ""
  local back = front:reverse():match("^%s*(.+)") or ""
  return back:reverse()
end

local function splitc(str, sep, ppairs)
  local parts = {}
  local pstack = {}
  local lasti = 1
  for i = 1, #str do
    local c = str:sub(i, i)
    if c == sep then
      if #pstack == 0 then
        parts[#parts + 1] = str:sub(lasti, i - 1)
        lasti = i + 1
      end
    elseif c == pstack[#pstack] then
      -- Pop paren
      pstack[#pstack] = nil
    else
      for k, v in pairs(ppairs) do
        if c == k then
          pstack[#pstack + 1] = v
        elseif c == v then
          -- Unbalanced
          error(("{%d} Unbalanced expression: %s"):format(env.rip, str), 0)
        end
      end
    end
  end

  parts[#parts + 1] = str:sub(lasti, #str)

  return parts
end

local function prepare(code)
  return setfenv(load(code), env)
end

local function eval(expr)
  if expr:match("^%d+$") then
    return tonumber(expr)
  elseif expr:match("^%%.+$") then
    local name = expr:match("%%(.+)")
    if env[name] then
      return tonumber(env[name])
    else
      error(("{%d} Unsupported eval name: %s"):format(env.rip, name), 0)
    end
  else
    error(("{%d} Unsupported eval: %s"):format(env.rip, expr), 0)
  end
end

local function address(expr)
  local offset = expr:match("^[+-]?%d+") or 0
  local inner = expr:match("%((.+)%)")
  local a, b, c, rest
  local parts = splitc(inner, ",", {})
  for i = 1, 3 do
    parts[i] = trim(parts[i] or "")
    if parts[i] == "" then
      parts[i] = nil
    end
  end

  return offset + eval(parts[1] or "0") + eval(parts[2] or "0") * eval(parts[3] or "1")
end

local function immedize(expr, bytes)
  bytes = bytes or 8
  
  if expr:match("^%$[+-]?%d+$") then
    return function() return tonumber(expr:match("%$(.+)")) end
  elseif expr:match("^%%.+$") then
    local name = expr:match("%%(.+)")
    if env[name] then
      return function() return env[name] end
    else
      error(("{%d} Unsupported immedization name: %s"):format(env.rip, name), 0)
    end
  elseif expr:match("%(.+%)") then
    local addr = address(expr)
    return function()
      return stack[addr]
    end
  else
    error(("{%d} Unsupported immedization: %s"):format(env.rip, expr), 0)
  end
end

local function dset(expr, bytes)
  bytes = bytes or 8

  if expr:match("%(.+%)") then
    local addr = address(expr)
    return function(v)
      stack[addr] = v
    end
  elseif expr:match("^%%.+$") then
    local name = expr:match("%%(.+)")
    if env[name] then
      return function(v) env[name] = v end
    else
      error(("{%d} Unsupported dset name: %s"):format(env.rip, name), 0)
    end
  else
    error(("{%d} Unsupported dset: %s"):format(env.rip, expr), 0)
  end
end

local instructions = {
  ret = function()
    if #labelStack == 1 then
      running = false
    else
      env.rip = stack[env.rsp]
      env.rsp = env.rsp + 8
      labelStack[#labelStack] = nil
    end
  end,
  movq = function(src, dest)
    local srcVal = immedize(src)()
    local destSetter = dset(dest)
    destSetter(srcVal)
  end,
  leaq = function(src, dest)
    local srcAddr = address(src)
    local destSetter = dset(dest)
    destSetter(srcAddr)
  end,
  pushq = function(src)
    local srcVal = immedize(src)()
    env.rsp = env.rsp - 8
    stack[env.rsp] = srcVal
  end,
  call = function(label)
    if labels[label] then
      env.rsp = env.rsp - 8
      stack[env.rsp] = env.rip

      env.rip = labels[label]
      labelStack[#labelStack + 1] = label
    else
      error(("{%d} Could not find label: %s"):format(env.rip, label), 0)
    end
  end,
  jmp = function(label)
    if labels[label] then
      env.rip = labels[label]
    else
      error(("{%d} Could not find label: %s"):format(env.rip, label), 0)
    end
  end,
  subq = function(sval, dest)
    local srcVal = immedize(sval)()
    local origVal = immedize(dest)()
    local destSetter = dset(dest)
    destSetter(origVal - srcVal)
  end,
  addq = function(sval, dest)
    local srcVal = immedize(sval)()
    local origVal = immedize(dest)()
    local destSetter = dset(dest)
    destSetter(origVal + srcVal)
  end,
  andq = function(mask, dest)
    local srcVal = immedize(mask)()
    local origVal = immedize(dest)()
    local destSetter = dset(dest)
    destSetter(bit.band(origVal, srcVal))
  end,
  leave = function()
    env.rsp = env.rbp
    env.rsp = env.rsp + 8
    env.rbp = stack[env.rsp]
  end,
  cmpq = function(src1, src2)
    local srcVal1 = immedize(src1)()
    local srcVal2 = immedize(src2)()
    local temp = tonumber(srcVal1) - tonumber(srcVal2)
    env.zf = temp == 0 and 1 or 0
    env.sf = temp >= 0 and 0 or 1
  end,
  jl = function(label)
    if labels[label] then
      if env.zf == 0 and env.sf == 0 then
        env.rip = labels[label]
      end
    else
      error(("{%d} Could not find label: %s"):format(env.rip, label), 0)
    end
  end
}

local function runCycle()
  local line = lines[tonumber(env.rip)]

  if not line:match("^.?%w+:") then
    -- Normal instruction
    line = trim(line)
    local instr, rest = line:match("(%S+)%s*(.*)$")

    local args = {}
    if rest then
      -- for arg in rest:gmatch("[^,]+") do
      --   args[#args + 1] = trim(arg)
      -- end
      args = splitc(rest, ",", {["("] = ")"})
      for i = 1, #args do
        args[i] = trim(args[i])
      end
    end

    if instructions[instr] then
      instructions[instr](unpack(args))
    else
      error(("{%d} Unsupported instruction: %s"):format(env.rip, instr), 0)
    end
  end

  env.rip = env.rip + 1
  line = lines[tonumber(env.rip)]
  while not line or line:match("^%s+$") do
    if env.rip > #lines then
      running = false
      return
    end

    env.rip = env.rip + 1
    line = lines[env.rip]
  end
end

local rspDrawOff = 5
local blockSize = 140
function _draw()
  cls()

  local index = 0
  for i = tonumber(env.rip) - 5, tonumber(env.rip) + 17 do
    if lines[i] then
      write(lines[i], 9, index * 7)
    end

    index = index + 1
  end

  write("=>", 0, 35)

  rectFill(blockSize, 0, _w, _h, 6)

  for i = 1, #order do
    write(order[i] .. ": " .. tohexstr(env[order[i]]), blockSize + 2, i * 7 - 7)
  end

  rectFill(_w - 62, 0, 62, _h, 16)
  index = 0
  for i = tonumber(env.rsp) - (rspDrawOff*8), tonumber(env.rsp) + (21*8) - (rspDrawOff*8), 8 do
    if i == tonumber(env.rsp) then
      write("rsp => ", _w - 60, index * 7, 1)
    else
      write(("%+d"):format(i - env.rsp), _w - 60, index * 7, 7)
    end
    write(("%4x"):format(i), _w - 85, index * 7)
    write(tohexstr(stack[i], 4), _w - 32, index * 7, 1)
    index = index + 1
  end

  if not running then
    write("EC: " .. tonumber(env.rax), blockSize + 2, _h - 8)
  end

  cursor:render()
  swap()
end

function _event(e, ...)
  cursor:event(e, ...)

  if e == "key" then
    local k = ...
    if k == "return" then
      if running then
        runCycle()
      end
    end
  elseif e == "mouseWheel" then
    local dir = ...
    rspDrawOff = rspDrawOff + dir
  end
end

1_p6.s

# some sample asm

  # long aframe(long n, long idx, long *q)
	# n in %rdi, idx in %rsi, q in %rdx
aframex:
    pushq	%rbp
    movq	%rsp, %rbp
    subq	$16, %rsp           # Allocate space for i (%rsp = s1)
    leaq	30(,%rdi,8), %rax   # 30 + n*8 => %rax
    andq	$-16, %rax          # %rax &= -16 (Sets LSB -> 0)
    subq	%rax, %rsp          # Allocate space for array p (%rsp = s2)  
                                # 1=>32, 2=>32, 3=>48, 4=>48, 5=>64
    leaq	15(%rsp), %r8
    andq	$-16, %r8           # Set %r8 to &p[0]
    ret # added

aframe:
    pushq   %rbp
    movq    %rsp, %rbp
    subq    $16, %rsp
    leaq    30(,%rdi,8), %rax
    andq    $-16, %rax
    subq    %rax, %rsp
    leaq    15(%rsp), %r8
    andq    $-16, %r8
    movq    %r8, %rcx
    leaq    -16(%rbp), %rax
    movq    %rax, (%r8)
    movq    $1, -16(%rbp)
    jmp     .L2
.L3:
    movq    %rdx, (%rcx,%rax,8)
    addq    $1, -16(%rbp)
.L2:
    movq    -16(%rbp), %rax
    cmpq    %rdi, %rax
    jl      .L3
    movq    (%r8,%rsi,8), %rax
    movq    (%rax), %rax
    leave
    ret

main:
    pushq   %rbp
    movq    %rsp, %rbp
    subq $8, %rsp      # allocate some space for a long
    movq $17, (%rsp)   # long = 17
    movq $5, %rdi   # n = 5
    movq $0, %rsi   # idx = 1
    movq %rsp, %rdx # q = &long

    call aframe

    addq $8, %rsp # deallocate space for long
    ret