PwnThyBytes 2019 CTF - unconventional

1_trace.py

"""
echo pass | TRACE=trace1 time gdb -x script.py -batch ./unconventional >/dev/null
~1 minute
"""
import gdb, re, os

gdb.execute('break *0x40542f')
gdb.execute('run')

f = open(os.environ["TRACE"], "w")
while True:
    for reg in "rax rbx rsp rdi rsi rdx rcx".split():
        out = gdb.execute('p/x $%s' % reg, to_string=True).strip()
        f.write("%s: " % reg + out + "\n")

    out = gdb.execute('info registers eflags', to_string=True).strip()
    f.write("flags: " + out + "\n")

    out = gdb.execute('x /i $pc', to_string=True).rstrip('\n')
    f.write(out + "\n")

    gdb.execute('stepi', to_string=False)
    gdb.flush()

2_hash.py

import gdb, re, os, sys

bp = [
    0x403eae, 0x4041fc, 0x404562, 0x4048c0,
    0x404c1e, 0x404f7c, 0x4052da
]
gdb.execute('break *0x40542f')
gdb.execute('run')
gdb.execute('hbreak *0x403eae')
gdb.execute("continue")

while True:
    pc = gdb.selected_frame().pc()
    if pc in bp:
        rbx = int(gdb.parse_and_eval('$rbx'))
        print("%08x" % rbx, file=sys.stderr, end="")
        if pc == bp[-1]:
            print(file=sys.stderr)
            break
    gdb.execute('si', to_string=False)
    gdb.flush()

3_solve.py

from __future__ import print_function, division
from subprocess import Popen, PIPE

def get(s):
    p = Popen(
        "gdb -x hash.py ./unconventional -batch",
        stderr=PIPE, stdout=PIPE, stdin=PIPE, shell=True)
    out, err = p.communicate(s)
    return err.strip()

target = int("7f7f7f805bdbd764fecac28069b5bd908ac68ad861819da67ffffffe", 16)
res = [0] * 32
for i in xrange(32):
    for j in xrange(8):
        s = list("\x00" * 32)
        s[i] = chr(1 << j)
        s = "".join(s)
        h = get(s)
        v = int(h, 16)
        print("%2d %d:" % (i, j), h.replace("0", "."))
        if target & v:
            res[i] |= 1 << j
            print(`"".join(map(chr, res))`)