Convert PSYQ .OBJ and .LIB to ELF and .a

psxlib.py

import os, os.path
import struct, array, sys, io

def readstring(f):
    return f.read(f.read(1)[0])

class SectionType:
    NULL = 0
    PROGBITS = 1
    SYMTAB = 2
    STRTAB = 3
    NOBITS = 8
    REL = 9
    INIT_ARRAY = 14
    FINI_ARRAY = 15

class SectionFlags:
    WRITE = 1 << 0
    ALLOC = 1 << 1
    EXECINSTR = 1 << 2
    INFO_LINK = 1 << 6

class NullSection:
    def __init__(self):
        self.name = None
        self.substitutions = []
        self.data = None
    def write(self, f):
        f.write(b'\0'*40)

class Section:
    def __init__(self, id, name):
        self.id = id
        self.name = name
        self.substitutions = []
        self.symbols = []
        self.data = bytes()
        self.empty = True
        self.externs = []
    def write(self, f):
        type, flags, align = {
            b'.text':  (SectionType.PROGBITS, SectionFlags.ALLOC|SectionFlags.EXECINSTR, 4),
            b'.ctors': (SectionType.INIT_ARRAY, 0, 4),
            b'.dtors': (SectionType.FINI_ARRAY, 0, 4),
            b'.data':  (SectionType.PROGBITS, SectionFlags.WRITE|SectionFlags.ALLOC, 1),
            b'.rdata': (SectionType.PROGBITS, SectionFlags.ALLOC, 1),
            b'.sdata': (SectionType.PROGBITS, SectionFlags.WRITE|SectionFlags.ALLOC, 1),
            b'.sbss':  (SectionType.NOBITS, SectionFlags.WRITE|SectionFlags.ALLOC, 1),
            b'.bss':   (SectionType.NOBITS, SectionFlags.WRITE|SectionFlags.ALLOC, 1)}[self.name]
        f.write(struct.pack('<IIIIIIIIII',
            self.nameoffset,
            type,
            flags,
            0, # addr
            self.dataoffset,
            len(self.data),
            0, # link
            0, # info
            align,
            0)) # ent size

class RelocSection:
    def __init__(self, orig):
        self.orig = orig
        self.name = b'.rel'+orig.name
    def resolveSym(self, symTab):
        self.data = b''
        for sub in self.orig.substitutions:
            offset = sub['offset']
            id = sub['data'][-2]
            if id not in symTab.idToIndex: continue # TODO
            sym = symTab.idToIndex[id]
            type = {0x10: 2, # R_MIPS_32
                    0x1A: 6, # R_MIPS_LO16?
                    0x1E: 6, # R_MIPS_LO16?
                    0x4A: 4, # R_MIPS_26
                    0x52: 5, # R_MIPS_HI16
                    0x54: 6  # R_MIPS_LO16
                    }[sub['insttype']]
            info = (sym << 8) + (type & 0xff)
            self.data += struct.pack('<II', offset, info)
    def write(self, f):
        f.write(struct.pack('<IIIIIIIIII',
            self.nameoffset,
            SectionType.REL,
            SectionFlags.INFO_LINK,
            0, # addr
            self.dataoffset,
            len(self.data),
            self.symTabIndex, # link
            self.orig.index, # info
            4, # align
            8)) # ent size

class SymTabSection:
    def __init__(self, sections):
        self.name = b'.symtab'
        self.substitutions = []
        self.data = b'\0'*16
        self.strings = []
        self.stringOffsets = []
        self.idToIndex = {}
        for secIdx, section in enumerate(sections):
            if not isinstance(section, Section) or section.empty:
                continue
            name = 0
            self.stringOffsets.append(len(self.data))
            self.strings.append(section.name)
            value = 0
            size = 0
            type = 3 # section
            bind = 0 # local
            info = (bind << 4) + (type & 0xf)
            other = 0 # vis default
            self.data += struct.pack('<IIIBBH', name, value, size, info, other, secIdx)
        for secIdx, section in enumerate(sections):
            if not isinstance(section, Section) or section.empty:
                continue
            for symbol in section.symbols:
                name = 0
                self.stringOffsets.append(len(self.data))
                self.strings.append(symbol['name'])
                self.idToIndex[symbol['id']] = len(self.strings)
                value = symbol['offset']
                size = 0
                type = 0 # notype
                bind = 0 # local
                info = (bind << 4) + (type & 0xf)
                other = 0 # vis default
                self.data += struct.pack('<IIIBBH', name, value, size, info, other, secIdx)
        for secIdx, section in enumerate(sections):
            if not isinstance(section, Section) or section.empty:
                continue
            for symbol in section.externs:
                name = 0
                self.stringOffsets.append(len(self.data))
                self.strings.append(symbol['name'])
                self.idToIndex[symbol['id']] = len(self.strings)
                value = 0
                size = 0
                type = 0 # notype
                bind = 1 # global
                info = (bind << 4) + (type & 0xf)
                other = 0 # vis default
                self.data += struct.pack('<IIIBBH', name, value, size, info, other, 0)
    def resolveStrings(self, strTab):
        for offsetOffset, strOffset in zip(self.stringOffsets, strTab.offsets):
            self.data = self.data[:offsetOffset]+struct.pack('<I', strOffset)+self.data[offsetOffset+4:]
    def write(self, f):
        f.write(struct.pack('<IIIIIIIIII',
            self.nameoffset,
            SectionType.SYMTAB,
            0, # flags
            0, # addr
            self.dataoffset,
            len(self.data),
            self.strTabIndex, # link
            0, # info
            4, # align
            16)) # ent size

class StrTabSection:
    def __init__(self, name, strings):
        self.name = name
        self.substitutions = []
        self.data = b'\0'
        self.offsets = []
        for s in strings:
            if s is None:
                continue
            self.offsets.append(len(self.data))
            self.data += s+b'\0'
    def write(self, f):
        f.write(struct.pack('<IIIIIIIIII',
            self.nameoffset,
            SectionType.STRTAB,
            0, # flags
            0, # addr
            self.dataoffset,
            len(self.data),
            0,
            0,
            1,
            0))

import binascii

class ObjFile:
    def __init__(self):
        self.sections = []
    def getSection(self, secId):
        return [section for section in self.sections if section.id == secId][0]
    def read(self, f):
        magic = f.read(4)
        assert magic == b'LNK\x02'
        assert f.read(2) == b'\x2e\x07'
        lastSection = None
        while True:
            c = f.read(1)
            assert len(c) > 0
            secType = c[0]
            if secType == 0:
                d = f.read()
                assert d == b'', d[:50]
                break
            elif secType == 0x06:
                secId, = struct.unpack('<H', f.read(2))
                currentSection = self.getSection(secId)
                print('Section data:', hex(secId))
                currentSection.empty = False
            elif secType == 0x02:
                l, = struct.unpack('<H', f.read(2))
                data = f.read(l)
                print('Data', l)
                #a = array.array('I')
                #a.frombytes(data)
                #if sys.byteorder == 'big': a.byteswap()
                #for i, x in enumerate(a):
                #    print('%x: %08x'%(i*4, x))
                currentSection.data = data
            elif secType == 0x0A:
                insttype, offset = struct.unpack('<BH', f.read(3))
                d = f.read(3)
                while d[-3] in (0x2E, 0x2C):
                    if d[-3] == 0x2E:
                        d += f.read(4)
                    elif d[-3] == 0x2C:
                        d += f.read(6)
                print('Substitute:', {0x52: 'lui  ', 0x54: 'addiu', 0x4a: 'j    ', 0x10: 'raw  '}.get(insttype, hex(insttype)), 'offset', hex(offset), binascii.b2a_hex(d).decode())
                currentSection.substitutions.append({'insttype': insttype, 'offset': offset, 'data': d})
            elif secType in (0x0C, 0x30):
                symId, secId, offset = struct.unpack('<HHI', f.read(8))
                symName = readstring(f)
                print('Symbol type', secType, ':', 'id', hex(symId), 'section', hex(secId), 'offset', hex(offset), symName)
                self.getSection(secId).symbols.append({'id': symId, 'offset': offset, 'name': symName})
                self.getSection(secId).empty = False
            elif secType == 0x0E:
                symId, = struct.unpack('<H', f.read(2))
                symName = readstring(f)
                #currentSection.externs.append({'id': symId, 'name': symName})
                # XXX
                self.getSection(secId).externs.append({'id': symId, 'name': symName})
                print('Extern: id', hex(symId), symName)
            elif secType == 0x10:
                secId, = struct.unpack('<H', f.read(2))
                d = f.read(3)
                secName = readstring(f)
                print('Section: id', hex(secId), d, secName)
                self.sections.append(Section(secId, secName))
            elif secType == 0x14:
                d = f.read(3)
                s = readstring(f)
                print('0x14', d, s)
            elif secType == 0x12:
                d = f.read(6)
                s = readstring(f)
                print('0x12', d, s)
            elif secType == 0x08:
                d = f.read(4)
                print('0x8', d)
            elif secType == 0x3C:
                d = f.read(2)
                print('0x3C', d)
            elif secType == 0x04:
                d = f.read(1)
                print('0x4', d)
            elif secType == 0x05:
                d = f.read(1)
                print('0x5', d)
            elif secType == 0x1c:
                d = f.read(2)
                s = readstring(f)
                print('0x9', d, s)
            else:
                print('Unknown section type', hex(secType), hex(f.tell()))
    def write(self, f):
        sections = [NullSection()]
        for section in self.sections:
            #if section.empty:
            #    continue
            section.index = len(sections)
            sections.append(section)
            if len(section.substitutions) > 0:
                sections.append(RelocSection(section))
        symTabIndex = len(sections)
        symTab = SymTabSection(sections)
        for section in sections:
            if isinstance(section, RelocSection):
                section.resolveSym(symTab)
        sections.append(symTab)
        strTabIndex = len(sections)
        strTab = StrTabSection(b'.strtab', symTab.strings)
        symTab.resolveStrings(strTab)
        sections.append(strTab)
        shStrTab = StrTabSection(b'.shstrtab', [section.name for section in sections]+[b'.shstrtab'])
        sections.append(shStrTab)
        f.write(b'\x7FELF')
        f.write(struct.pack('<BBBBB7xHHIIIIIHHHHHH',
            1, # 32-bit
            1, # little-endian
            1, # version
            0, # ABI
            0, # ABI version
            1, # relocatable
            8, # MIPS
            1, # version
            0, # entry point
            0, # PH offset
            0, # SH offset (added later)
            0x1000, # flags: o32, mips1
            52, # header size
            0, # PH size
            0, # PH num
            40, # SH size
            len(sections),
            len(sections)-1)) # section string table index
        
        for section in sections:
            if section.data is None:
                continue
            section.dataoffset = f.tell()
            f.write(section.data)
        i = 0
        for section in sections:
            if section.name is None:
                continue
            section.nameoffset = shStrTab.offsets[i]
            section.symTabIndex = symTabIndex
            section.strTabIndex = strTabIndex
            i += 1
        
        sectionHeaderOffset = f.tell()
        f.seek(0x20)
        f.write(struct.pack('<I', sectionHeaderOffset))
        f.seek(sectionHeaderOffset)
        for section in sections:
            section.write(f)

def writeArchive(fout, symbols, symbolIndices, elfs):
    fout.write(b'!<arch>\n')
    # symbol table
    fout.write(b'/'.ljust(16))
    fout.write(b'0'.ljust(12))#fout.write(str(int(stat.st_mtime)).encode().ljust(12))
    fout.write(b'0'.ljust(6)) # owner ID
    fout.write(b'0'.ljust(6)) # group ID
    fout.write(b'0'.ljust(8)) # file mode
    calcTableLen = 4+len(symbols)*4+sum([len(symbol)+1 for symbol in symbols])
    fout.write(str(calcTableLen+(calcTableLen%2)).encode().ljust(10))
    fout.write(b'\x60\x0A')
    fout.write(struct.pack('>I', len(symbols)))
    fout.seek(4*len(symbols), 1) # do offsets later
    for symbol in symbols:
        fout.write(symbol)
        fout.write(b'\0')
    if calcTableLen%2 != 0:
        fout.write(b'\0')
    
    objPos = []
    for name, elf in elfs:
        objPos.append(fout.tell())
        fout.write((name.strip()+b'/').ljust(16))
        fout.write(b'0'.ljust(12))#fout.write(str(int(stat.st_mtime)).encode().ljust(12))
        fout.write(b'0'.ljust(6)) # owner ID
        fout.write(b'0'.ljust(6)) # group ID
        fout.write(b'644'.ljust(8)) # file mode
        fout.write(str(len(elf)).encode().ljust(10))
        fout.write(b'\x60\x0A')
        fout.write(elf)
    
    fout.seek(8+60+4)
    for objIdx in symbolIndices:
        fout.write(struct.pack('>I', objPos[objIdx]))

import subprocess, tempfile

for fname in sys.argv[1:]:
    print(fname)
    basename = os.path.splitext(fname)[0]
    f = open(fname, 'rb')
    stat = os.fstat(f.fileno())
    magic = f.read(4)
    if magic == b'LIB\x01':
        symbols = []
        symbolIndices = []
        elfs = []
        i = 0
        while True:
            startpos = f.tell()
            d = f.read(0x14)
            if d == b'': break
            name, check, headerlen, totallen = struct.unpack('<8sIII', d)
            #name = id.decode().strip()
            decname = name.decode().strip()+".OBJ"
            print(decname)
            while True:
                s = readstring(f).strip(b'\0')
                if s == b'': break
                symbols.append(s)
                symbolIndices.append(i)
            #print(symbols)
            assert f.tell() == startpos+headerlen
            lnk = f.read(totallen-headerlen)
            #o = ObjFile()
            #o.read(io.BytesIO(lnk))
            #oout = io.BytesIO()
            #o.write(oout)
            #if len(oout.getvalue())%2 != 0:
            #    oout.write(b'\0')
            #elfs.append((name, oout.getvalue()))
            tmpout = tempfile.NamedTemporaryFile()
            tmpout.write(lnk)
            tmpout.flush()
            tmpdir = tempfile.TemporaryDirectory()
            subprocess.run(["psyq-obj-parser", tmpout.name, "-n", "-o", decname], check=True, cwd=tmpdir.name, stdout=subprocess.DEVNULL)
            elfs.append((name, open(tmpdir.name+'/'+decname, 'rb').read()))
            i += 1
        f.close()
        
        fout = open(basename.lower()+'.a', 'wb')
        writeArchive(fout, symbols, symbolIndices, elfs)
        fout.close()
    elif magic == b'LIB\x02':
        symbols = []
        symbolIndices = []
        
        somepos, somelen = struct.unpack('<II', f.read(8))
        startpos = f.tell()
        f.seek(somepos)
        i = 0
        objs = []
        while f.tell() < somepos+somelen:
            objs.append(struct.unpack('<III', f.read(12))+(readstring(f),))
            while True:
                cmd, = struct.unpack('<H', f.read(2))
                if cmd == 0:
                    break
                elif cmd in (256, 512):
                    f.read(2).hex()
                    s = readstring(f)
                    assert b'\0' not in s
                    symbols.append(s)
                    symbolIndices.append(i)
                else:
                    assert False
            i += 1
        
        elfs = []
        for objpos, objlen, check, objname in objs:
            f.seek(objpos)
            lnk = f.read(objlen)
            #o = ObjFile()
            #o.read(io.BytesIO(lnk))
            #oout = io.BytesIO()
            #o.write(oout)
            #if len(oout.getvalue())%2 != 0:
            #    oout.write(b'\0')
            #elfs.append((name, oout.getvalue()))
            tmpout = tempfile.NamedTemporaryFile()
            tmpout.write(lnk)
            tmpout.flush()
            tmpdir = tempfile.TemporaryDirectory()
            decname = objname.decode()
            print(decname)
            try: subprocess.run(["psyq-obj-parser", tmpout.name, "-n", "-o", decname], check=True, cwd=tmpdir.name, stdout=subprocess.DEVNULL)
            except subprocess.CalledProcessError:
                elfs.append((objname, b''))
                continue
            try: elfs.append((objname, open(tmpdir.name+'/'+decname, 'rb').read()))
            except FileNotFoundError: elfs.append((objname, b''))
        
        fout = open(basename.lower()+'.a', 'wb')
        writeArchive(fout, symbols, symbolIndices, elfs)
        fout.close()
    elif magic == b'LNK\x02':
        f.seek(0)
        o = ObjFile()
        o.read(f)
        f.close()
        fout = open(basename+'.o', 'wb')
        o.write(fout)
        fout.close()
    else:
        assert False, magic
    print()