Skip to content

Instantly share code, notes, and snippets.

@mkow

mkow/rar4_vm_asm.py

Created September 24, 2019 15:15
Show Gist options
  • Save mkow/d9c6a74fac5749698ecebeafc8d59d70 to your computer and use it in GitHub Desktop.
Save mkow/d9c6a74fac5749698ecebeafc8d59d70 to your computer and use it in GitHub Desktop.
Download ZIP
Assembler for RAR v4 VM (anti-antivirus challenge, Real World CTF 2019 Quals)
Raw
rar4_vm_asm.py
import sys
VMCF_OP0 = 0
VMCF_OP1 = 1
VMCF_OP2 = 2
VMCF_OPMASK = 3
VMCF_BYTEMODE = 4
VMCF_JUMP = 8
VMCF_PROC = 16
VMCF_USEFLAGS = 32
VMCF_CHFLAGS = 64
vm_cmdflags = [
VMCF_OP2 | VMCF_BYTEMODE , # VM_MOV
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_CMP
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_ADD
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_SUB
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JZ
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JNZ
VMCF_OP1 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_INC
VMCF_OP1 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_DEC
VMCF_OP1 | VMCF_JUMP , # VM_JMP
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_XOR
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_AND
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_OR
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_TEST
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JS
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JNS
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JB
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JBE
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JA
VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS , # VM_JAE
VMCF_OP1 , # VM_PUSH
VMCF_OP1 , # VM_POP
VMCF_OP1 | VMCF_PROC , # VM_CALL
VMCF_OP0 | VMCF_PROC , # VM_RET
VMCF_OP1 | VMCF_BYTEMODE , # VM_NOT
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_SHL
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_SHR
VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_SAR
VMCF_OP1 | VMCF_BYTEMODE | VMCF_CHFLAGS , # VM_NEG
VMCF_OP0 , # VM_PUSHA
VMCF_OP0 , # VM_POPA
VMCF_OP0 | VMCF_USEFLAGS , # VM_PUSHF
VMCF_OP0 | VMCF_CHFLAGS , # VM_POPF
VMCF_OP2 , # VM_MOVZX
VMCF_OP2 , # VM_MOVSX
VMCF_OP2 | VMCF_BYTEMODE , # VM_XCHG
VMCF_OP2 | VMCF_BYTEMODE , # VM_MUL
VMCF_OP2 | VMCF_BYTEMODE , # VM_DIV
VMCF_OP2 | VMCF_BYTEMODE | VMCF_USEFLAGS | VMCF_CHFLAGS , # VM_ADC
VMCF_OP2 | VMCF_BYTEMODE | VMCF_USEFLAGS | VMCF_CHFLAGS , # VM_SBB
VMCF_OP0 # VM_PRINT
]
VM_MOV = 0
VM_CMP = 1
VM_ADD = 2
VM_SUB = 3
VM_JZ = 4
VM_JNZ = 5
VM_INC = 6
VM_DEC = 7
VM_JMP = 8
VM_XOR = 9
VM_AND = 10
VM_OR = 11
VM_TEST = 12
VM_JS = 13
VM_JNS = 14
VM_JB = 15
VM_JBE = 16
VM_JA = 17
VM_JAE = 18
VM_PUSH = 19
VM_POP = 20
VM_CALL = 21
VM_RET = 22
VM_NOT = 23
VM_SHL = 24
VM_SHR = 25
VM_SAR = 26
VM_NEG = 27
VM_PUSHA = 28
VM_POPA = 29
VM_PUSHF = 30
VM_POPF = 31
VM_MOVZX = 32
VM_MOVSX = 33
VM_XCHG = 34
VM_MUL = 35
VM_DIV = 36
VM_ADC = 37
VM_SBB = 38
VM_PRINT = 39
class Op(object):
pass
class RegOp(Op):
def __init__(self, reg):
self.reg = reg
class IntOp(Op):
def __init__(self, val):
self.val = val
class RegMemOp(Op):
def __init__(self, reg, base):
self.reg = reg
self.base = base
class BitStream(object):
def __init__(self, data, start_bitpos):
self.data = data
self.pos = start_bitpos
def write_bits(self, bits, cnt):
assert (bits >> cnt) == 0
for i in xrange(cnt):
masked = (self.data[self.pos / 8] & ~(1<<(7-self.pos%8)))
self.data[self.pos / 8] = masked | (((bits >> (cnt-i-1)) & 1) << (7-self.pos%8))
self.pos += 1
def rarvm_write_data(self, value):
if value < 16:
self.write_bits(0, 2)
self.write_bits(value, 4)
elif 0xffffff00 < value <= 0xffffffff:
self.write_bits(1, 2)
self.write_bits(0, 4)
self.write_bits(value - 0xffffff00, 8)
elif value < 256:
assert (value >> 4) != 0
self.write_bits(1, 2)
self.write_bits(value, 8)
elif value < 2**16:
self.write_bits(2, 2)
self.write_bits(value, 16)
elif value < 2**32:
self.write_bits(3, 2)
self.write_bits(value >> 16, 16)
self.write_bits(value & 0xFFFF, 16)
else:
raise RuntimeError('Can\'t encode this value!')
def insert_insn(self, op, byte_mode, opnds):
assert op >= 0
if op < 0b1000:
self.write_bits(op, 4)
else:
assert ((op + 24) >> 5) == 1 # weird, but that's how the parser works
self.write_bits(op + 24, 6)
if vm_cmdflags[op] & VMCF_BYTEMODE:
self.write_bits(byte_mode, 1)
else:
assert byte_mode == 0
op_num = vm_cmdflags[op] & VMCF_OPMASK
assert len(opnds) == op_num
for i in xrange(op_num):
if isinstance(opnds[i], RegOp):
self.write_bits(1, 1)
self.write_bits(opnds[i].reg, 3)
elif isinstance(opnds[i], IntOp):
self.write_bits(0, 2)
if byte_mode == 1:
self.write_bits(opnds[i].val, 8)
else:
self.rarvm_write_data(opnds[i].val)
else:
assert isinstance(opnds[i], RegMemOp)
self.write_bits(1, 2)
if opnds[i].base == 0:
self.write_bits(0, 1)
self.write_bits(opnds[i].reg, 3)
else:
self.write_bits(1, 1)
# one (impossible?) case skipped here, we always write '0'
self.write_bits(0, 1)
self.write_bits(opnds[i].reg, 3)
self.rarvm_write_data(opnds[i].base)
def main(argv):
with open(argv[1], 'rb') as f:
data = f.read()
data = bytearray(data)
bs = BitStream(data, (0x7f + 0x92) * 8 + 2) # first VM in test.rar
# read_vm_code()
bs.write_bits(0, 1) # has filters
bs.write_bits(0, 1) # block_start += 258
bs.write_bits(1, 1) # has block_length
bs.write_bits(0, 1) # has init_mask for stack_filter->prg.init_r initialization
bs.write_bits(0, 1) # has global_data initializer
vm_code_size = 1337 # no idea how is this different from vm_codesize
assert vm_code_size > 0
if vm_code_size < 7:
bs.write_bits(vm_code_size - 1, 3)
elif vm_code_size <= 255+7:
bs.write_bits(7-1, 3)
bs.write_bits(vm_code_size - 7, 8)
else:
assert vm_code_size < 2**16
bs.write_bits(8-1, 3)
bs.write_bits(vm_code_size, 16)
# add_vm_code()
block_start = 0
bs.rarvm_write_data(block_start)
block_length = 200
bs.rarvm_write_data(block_length)
vm_code = bytearray('aaaaaaaaaaaaaaa') # size needs to be adjusted for longer code
vm_bs = BitStream(vm_code, 0)
data_flag = 0 # has prg->static_data initializer?
vm_bs.write_bits(data_flag, 1)
vm_bs.insert_insn(VM_MOV, 0, [RegOp(0), IntOp(0)])
vm_bs.insert_insn(VM_MOV, 0, [RegMemOp(0, 0), IntOp(0x1337)]) # [r0+0] = 0x1337
vm_bs.insert_insn(VM_MOV, 0, [RegOp(1), RegMemOp(0, 0)])
vm_bs.insert_insn(VM_PUSH, 0, [RegOp(0)])
vm_bs.insert_insn(VM_POP, 0, [RegOp(0)])
vm_bs.insert_insn(VM_PUSHA, 0, [])
vm_bs.insert_insn(VM_POPA, 0, [])
vm_bs.insert_insn(VM_PRINT, 0, [])
vm_bs.insert_insn(VM_RET, 0, [])
vm_code = bytearray([reduce(lambda x,y: x^y, vm_code)]) + vm_code # prepend checksum
bs.rarvm_write_data(len(vm_code))
for b in vm_code:
bs.write_bits(b, 8)
#global_data = 'asdasdsadasdsad'
#bs.rarvm_write_data(len(global_data))
#for i in xrange(len(global_data)):
# bs.write_bits(ord(global_data[i]), 8)
with open(argv[2], 'wb') as f:
f.write(data)
if __name__ == '__main__':
main(sys.argv)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment