Parser for Darkest Dungeon saves

parser.py

from struct import unpack, pack
from sys import argv, stderr
import io
import json

def checked_read(f, amount):
  result = f.read(amount)
  assert len(result) == amount
  return result

def parse_document(f):
  header, _, _, _, _, table_count, _, _, _, _, _, triplets_count, _, _, _, data_start = unpack('<' + 'I'*16, f.read(4*16))
  assert header == 0xB101
  table = [ unpack('<IIII', f.read(16)) for _ in range(table_count) ]
  tri = [ unpack('<III', f.read(12)) for _ in range(triplets_count) ]
  assert f.tell() == data_start
  data = [ checked_read(f, i2[1] - i1[1]) for i1, i2 in zip(tri, tri[1:]) ]
  data.append( f.read() )

  def parse_value(pos, value):
    if value == b'\x00':
      return False
    if value == b'\x01':
      return True
    # consume 32-bit alignment
    assert len(value) >= (-pos % 4) # and all(x == 0 for x in value[:(-pos % 4)])
    value = value[(-pos % 4):]
    # uint
    uint, value = unpack('<I', value[:4])[0], value[4:]
    if not value: return uint
    # string
    assert len(value) >= uint
    if value.startswith(b'\x01\xb1\x00\x00'):  # persist.roster.json contains sub-documents within fields
      return parse_document(io.BytesIO(value))
    assert value[uint-1] == 0
    return value[:uint-1].decode()

  nodes = []
  for d, (u, pos, flags) in zip(data, tri):
    # Separate flags
    t = bool(flags >> 31)
    tableSlot = (flags & ((1<<31)-1))>>11
    nameSize = (flags & ((1<<11)-1))>>2
    f1 = bool(flags & 2)
    f0 = bool(flags & 1)
    assert not f1

    # Separate data into name and value
    assert d[nameSize-1] == 0
    name, value = d[:nameSize-1].decode(), d[nameSize:]

    # Build result
    node = { 'u': u, 't': t, 'name': name }
    if f0:
      assert not value
      node['tableSlot'] = tableSlot
    else:
      assert not tableSlot
      try:
        node['value'] = parse_value(pos + nameSize, value)
      except Exception as e:
        node['value'] = 'ERROR'
        print('Error parsing {}: {}'.format(value, repr(e)), file=stderr)
    nodes.append(node)

  # Process nodes, building object tree
  from collections import OrderedDict
  stack = [(0xffffffff, OrderedDict(), float('inf'))]
  for idx, node in enumerate(nodes):
    target = stack[-1][1]
    if 'tableSlot' in node:
      parent, triplet, children, totalChildren = table[node['tableSlot']]
      assert parent == stack[-1][0] and triplet == idx
      value = OrderedDict()
      stack.append((node['tableSlot'], value, children))
    else:
      value = node['value']
    assert node['name'] not in target
    target[node['name']] = value
    while len(stack[-1][1]) == stack[-1][2]: stack.pop()
  assert len(stack) == 1
  result = stack[0][1]
  assert len(result) == 1 and 'base_root' in result
  result = result['base_root']
  return result

result = parse_document(open(argv[1], 'rb'))
print(json.dumps(result, ensure_ascii=False, indent=4))