mildsunrise/dump.py

## dump.py
# this tool does the following:
#  - dump the atlas of a gnome-shell process (its backing texture, as well as the allocations in it).
#  - cross-reference the info with the state of the glyph cache and the texture cache (to determine the source of some of the allocations).
#  - perform a number of consistency checks on all of that, in the form of assertions,
#  - finally, plot the data in a nice figure.
#
# allocations are tinted with a color depending on their status / source:
#   - grayish → free space1
#   - yellow → glyph cache
#   - red → texture cache, but unused
#   - cyan → texture cache, used
#   - blue → other
#
# hovering over a rectangle (region of the atlas) tells you:
#   - the bounds of that rectangle: "(x, y), (width, height)" in pixels
#   - in case the rectangle is allocated:
#     - refs to the AtlasTexture exposing the rectangle (only shown if != 1): "(2 refs)"
#     - for glyph cache allocations: font + glyph
#     - for texture cache allocations: key string + additional refs to the ClutterImage exposing it
#       (i.e. not counting the ref from the texture cache itself)
#
# the tool can operate on live processes (reading /proc/x/mem + attaching with gdb to download
# textures) or it can operate on corefiles. (in which case it can use textures if previously
# downloaded using the GDB script, and saved as `<corefile>.texture.<address in decimal>`).
#
# it may surely fail due to race conditions. I'm reading memory from a live process after all.
# try running it again in that case.
#
# the code quality is bad. this was hacked for my needs (version 41.2) so don't expect to be
# able to use it easily. also I didn't have symbols which means I have literally hardcoded
# the layout of A WHOLE LOT OF C STRUCTS below. it will probably not match yours, especially
# CoglContext which is internal and updated quite often. there's even hardcoded symbol
# addresses and you'll have to update those as well. you've been warned.
#
# (if you *really* need this to work, it wouldn't be *that* much time for me to adapt it to
# use debug symbols to deduce all it needs (structs, global variables) but I can't guarantee
# anything)
#
# dependencies:
#  - py-struct: https://github.com/mildsunrise/py-struct
#  - gdb (only if working on a live process)
#  - emucore (only if working on a corefile)
#  - numpy + matplotlib (for the plotting part)
#
# scroll a bit and enter your parameters...


from typing import Annotated, BinaryIO, Iterator, NamedTuple, Optional, TypeVar, Union, cast

from serialization.serialization import *
from dataclasses import dataclass
import enum
from struct import unpack

from pprint import PrettyPrinter
from subprocess import run
import re
import os

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle

def read_str(st: BinaryIO, pos: int) -> str:
    st.seek(pos)
    result = bytearray()
    while (x := st.read(1)[0]):
        assert len(result) < 4096
        result.append(x)
    return result.decode('utf-8')

Bool = U8
Char = S8; UChar = U8
Short = S16; UShort = U16
Int = S32; UInt = U32
Long = S64; ULong = U64
IntPtr = S64; UIntPtr = U64
Ptr = Size = U64


# KERNEL UTILS

class ProcfsMapEntry(NamedTuple):
    start: Ptr
    end: Ptr
    prot: str
    pgoff: Ptr
    dev: tuple[int, int]
    inode: int
    filename: Optional[bytes]

    @staticmethod
    def parse(line: bytes) -> 'ProcfsMapEntry':
        assert line[-1:] == b'\n'
        line = line[:-1]

        m = re.match(rb'([\da-f]{8,})-([\da-f]{8,}) ([r-][w-][x-][sp]) ([\da-f]{8,}) ([\da-f]{2,}:[\da-f]{2,}) (\d+) ', line)
        if not m: raise ValueError()
        start, end, prot, pgoff, dev, inode = m.groups()
        dev = tuple(int(x, 16) for x in dev.decode().split(':'))
        prot = prot.decode()

        if len(line) == m.end(0):
            fname = None
        else:
            bpos = max(m.end(0), 72) + 1  # FIXME: this is 48 on 32-bit machines
            assert len(line) >= bpos and all(x == ord(' ') for x in line[m.end(0):bpos])
            fname = line[bpos:]

        return ProcfsMapEntry(int(start, 16), int(end, 16), prot, int(pgoff, 16), dev, int(inode, 10), fname)

def pidof(name: str) -> list[int]:
    out = run(['pidof', name], check=True, capture_output=True, text=True)
    return list(map(int, out.stdout.split()))


# PRIMARY PARAMETERS
# [[ PUT YOURS HERE ]]

PROC_SRC: Union[int, str]
#   if working on a live process, it's set to an int (its PID).
#   if working on a corefile, PROC_SRC is set to a string (its filename).

FONT_MAP: Ptr
#   address (pointer) of the CoglPangoFontMap / PangoCairoFontMap instance.
#   from this root address, everything else is looked up.
#   - if it's a live session, you can easily get this address by using
#     the `probe_to_get_font_map.sh` script (see there for instructions).
#   - if it's a coredump it's much harder but it's possible by searching
#     the memory for pointers to the PangoCairoFontMap class, look at the
#     twitter thread

PROC_SRC, = pidof('gnome-shell'); FONT_MAP = 94419581069472
#PROC_SRC = 'core.2310'; FONT_MAP = 0x56454419c8f0
PROC_SRC = 'core.16791'; FONT_MAP = 94891084519664


# from here it's mostly C structs definitions mixed with some useful logic.
# you can safely skip to the MAIN LOGIC section.

# GLIB

GBool = Int
GQuark = U32
GDestroyNotify = Ptr

GHashFunc = GEqualFunc = Ptr
gatomicrefcount = Int

@dataclass
class GSList(Struct):
    data: Ptr
    next: Ptr # GSList

    @staticmethod
    def read(obj: Ptr, st: BinaryIO) -> Iterator[Ptr]:
        while obj:
            st.seek(obj)
            entry = GSList.__load__(st)
            yield entry.data
            obj = entry.next

@dataclass
class GQueue(Struct):
  head: Ptr # GList
  tail: Ptr # GList
  length: UInt

@dataclass
class GHashTable(Struct):
    size: Size
    mod: Int
    mask: UInt
    nnodes: Int
    noccupied: Int  # nnodes + tombstones

    flags: U8  # have_big_keys, have_big_values

    keys: Ptr
    hashes: Ptr # UInt
    values: Ptr

    hash_func: GHashFunc
    key_equal_func: GEqualFunc
    ref_count: gatomicrefcount

    # Tracks the structure of the hash table, not its contents: is only
    # incremented when a node is added or removed (is not incremented
    # when the key or data of a node is modified).
    version: Int  # FIXME: omitted ifdef G_DISABLE_ASSERT

    key_destroy_func: GDestroyNotify
    value_destroy_func: GDestroyNotify

    def read(self, st: BinaryIO, use_small_arrays: bool=False) -> dict[Ptr, Ptr]:
        def read_array(array: Ptr, is_big: bool) -> list[Ptr]:
            size, fmt = (8, 'Q') if is_big else (4, 'I')
            st.seek(array)
            return [ unpack(fmt, st.read(size))[0] for _ in range(self.size) ]
        read_big_array = lambda array, is_big: read_array(array, is_big or (not use_small_arrays))
        hashes = read_array(self.hashes, False)
        keys = read_big_array(self.keys, bool((self.flags >> 0) & 1))
        values = read_big_array(self.values, bool((self.flags >> 1) & 1))
        entries = [ (k, v) for h, k, v in zip(hashes, keys, values) if h >= 2 ]
        result = dict(entries)
        assert len(result) == len(entries) == self.nnodes
        return result

GHookFinalizeFunc = Ptr

@dataclass
class GHookList(Struct):
    seq_id: ULong
    hook_size: U16
    is_setup: Bool
    hooks: Ptr # GHook
    dummy3: Ptr
    finalize_hook: GHookFinalizeFunc
    dummy: Annotated[tuple[Ptr, ...], FixedSize(2)]

@dataclass
class GDataElt(Struct):
    key: GQuark
    data: Ptr
    destroy: GDestroyNotify

@dataclass
class GData(Struct):
    len: U32      # Number of elements
    alloc: U32    # Number of allocated elements
    # array of GDataElt follows...

@dataclass
class GTypeInstance(Struct):
    # < private >
    g_class: Ptr # GTypeClass

@dataclass
class GObject(Struct):
    parent_instance: GTypeInstance

    # < private >
    ref_count: UInt
    qdata: Ptr # GData

    QDATA_FLAGS = 0x7

    def get_qdata(self, st: BinaryIO, quark: GQuark) -> Ptr:
        st.seek(self.qdata & ~self.QDATA_FLAGS)
        dataset = GData.__load__(st)
        els = (GDataElt.__load__(st) for _ in range(dataset.len))
        return next(el.data for el in els if el.key == quark)

# PANGO

PangoDirection = Int
PangoGravity = Int
PangoGravityHint = Int
PangoUnderline = Int

@dataclass
class PangoContext(Struct):
    parent_instance: GObject
    serial: UInt
    fontmap_serial: UInt

    set_language: Ptr # PangoLanguage
    language: Ptr # PangoLanguage
    base_dir: PangoDirection
    base_gravity: PangoGravity
    resolved_gravity: PangoGravity
    gravity_hint: PangoGravityHint

    font_desc: Ptr # PangoFontDescription

    matrix: Ptr # PangoMatrix

    font_map: Ptr # PangoFontMap

    metrics: Ptr # PangoFontMetrics

    round_glyph_positions: GBool

PangoFontMap = GObject  # FIXME

@dataclass
class PangoRenderer(Struct):
    # < private >
    parent_instance: GObject

    underline: PangoUnderline
    strikethrough: GBool
    active_count: Int

    # < public >
    matrix: Ptr # PangoMatrix          # May be NULL

    # < private >
    priv: Ptr # PangoRendererPrivate


PangoGlyph = U32

# PANGO-CAIRO

PangoCairoFontMap = GObject # FIXME

# COGL

CoglBool = Int
CoglBitmask = IntPtr
CoglHandle = Ptr

COGL_BUFFER_BIND_TARGET_COUNT = 4
_COGL_N_FEATURE_IDS = 26
COGL_N_PRIVATE_FEATURES = 31
COGL_FLAGS_N_LONGS_FOR_SIZE = lambda n: (n + 63) // 64

GLint = Int; GLuint = UInt; GLenum = Int

CoglUserDataDestroyInternalCallback = Ptr

@dataclass
class CoglUserDataEntry(Struct):
    key: Ptr # CoglUserDataKey
    user_data: Ptr
    destroy: CoglUserDataDestroyInternalCallback

COGL_OBJECT_N_PRE_ALLOCATED_USER_DATA_ENTRIES = 2

@dataclass
class CoglObject(Struct):
    parent_instance: GTypeInstance

    user_data_entry: Annotated[list[CoglUserDataEntry], FixedSize(COGL_OBJECT_N_PRE_ALLOCATED_USER_DATA_ENTRIES)]
    user_data_array: Ptr # GArray
    n_user_data_entries: Int

    ref_count: UInt

CoglDepthTestFunction = Int
CoglFeatureFlags = Int
CoglColorMask = Int
CoglWinsysRectangleState = Int
CoglPipelineProgramType = Int
CoglFogMode = Int
CoglOffscreenAllocateFlags = Int
CoglDriver = Int
CoglMatrixMode = Int
CoglMatrixOp = Int

@dataclass
class CoglColor(Struct):
    red: U8
    green: U8
    blue: U8

    alpha: U8

    # padding in case we want to change to floats at some point
    padding0: U32
    padding1: U32
    padding2: U32

@dataclass
class CoglMatrix(Struct):
    __align__ = 16  # FIXME: not present in newer versions
    data: Annotated[list[Annotated[list[F32], FixedSize(4)]], FixedSize(4)]

    # < private >

    # FIXME: only present in newer versions
    ### Note: we may want to extend this later with private flags and a cache of the inverse transform matrix. */
    ##inv: Annotated[list[Annotated[list[F32], FixedSize(4)]], FixedSize(4)]
    ##type: ULong
    ##flags: ULong
    ##_padding3: ULong

@dataclass
class CoglMatrixEntry(Struct):
    parent: Ptr # CoglMatrixEntry
    op: CoglMatrixOp
    ref_count: UInt

    # FIXME: only ifdef COGL_DEBUG_ENABLED
    # used for performance tracing
    composite_gets: Int

@dataclass
class CoglMatrixEntryCache(Struct):
    entry: Ptr # CoglMatrixEntry
    flushed_identity: CoglBool
    flipped: CoglBool

@dataclass
class CoglList(Struct):
    prev: Ptr # CoglList
    next: Ptr # CoglList

CoglGpuInfoVendor = Int
CoglGpuInfoDriverPackage = Int
CoglGpuInfoArchitecture = Int
CoglGpuInfoArchitectureFlag = Int
CoglGpuInfoDriverBug = Int

@dataclass
class CoglGpuInfo(Struct):
    vendor: CoglGpuInfoVendor
    vendor_name: Ptr # string

    driver_package: CoglGpuInfoDriverPackage
    driver_package_name: Ptr # string
    driver_package_version: Int

    architecture: CoglGpuInfoArchitecture
    architecture_name: Ptr # string
    architecture_flags: CoglGpuInfoArchitectureFlag

    driver_bugs: CoglGpuInfoDriverBug

@dataclass
class CoglPipelineFogState(Struct):
    enabled: CoglBool
    color: CoglColor
    mode: CoglFogMode
    density: F32
    z_near: F32
    z_far: F32

# very unstable struct.
# based on https://gitlab.gnome.org/GNOME/mutter/-/blob/41.2/cogl/cogl/cogl-context-private.h
# lines prefixed with ## are from newer versions
@dataclass
class CoglContext(Struct):
    parent_instance: CoglObject

    display: Ptr # CoglDisplay

    driver: CoglDriver

    ### Information about the GPU and driver which we can use to determine certain workarounds
    ##gpu: CoglGpuInfo

    # vtables for the driver functions
    driver_vtable: Ptr # const CoglDriverVtable
    texture_driver: Ptr # const CoglTextureDriver

    driver_context: Ptr  # FIXME: no longer present

    glsl_major: Int
    glsl_minor: Int

    glsl_version_to_use: Int

    # Features cache
    features: Annotated[list[ULong], FixedSize(COGL_FLAGS_N_LONGS_FOR_SIZE (_COGL_N_FEATURE_IDS))]
    ##feature_flags: CoglFeatureFlags   # legacy/deprecated feature flags
    private_features: Annotated[list[ULong], FixedSize(COGL_FLAGS_N_LONGS_FOR_SIZE (COGL_N_PRIVATE_FEATURES))]

    ##needs_viewport_scissor_workaround: CoglBool
    ##viewport_scissor_workaround_framebuffer: Ptr # CoglFramebuffer

    default_pipeline: Ptr # CoglPipeline
    default_layer_0: Ptr # CoglPipelineLayer
    default_layer_n: Ptr # CoglPipelineLayer
    dummy_layer_dependant: Ptr # CoglPipelineLayer

    attribute_name_states_hash: Ptr # GHashTable
    attribute_name_index_map: Ptr # GArray
    n_attribute_names: Int

    ##enabled_builtin_attributes: CoglBitmask
    ##enabled_texcoord_attributes: CoglBitmask
    enabled_custom_attributes: CoglBitmask

    # These are temporary bitmasks that are used when disabling
    # builtin,texcoord and custom attribute arrays. They are here just
    # to avoid allocating new ones each time
    ##enable_builtin_attributes_tmp: CoglBitmask
    ##enable_texcoord_attributes_tmp: CoglBitmask
    enable_custom_attributes_tmp: CoglBitmask
    changed_bits_tmp: CoglBitmask

    legacy_backface_culling_enabled: CoglBool

    # A few handy matrix constants
    identity_matrix: CoglMatrix
    y_flip_matrix: CoglMatrix

    ### Value that was last used when calling glMatrixMode to avoid calling it multiple times
    ##flushed_matrix_mode: CoglMatrixMode

    # The matrix stack entries that should be flushed during the next pipeline state flush
    current_projection_entry: Ptr # CoglMatrixEntry
    current_modelview_entry: Ptr # CoglMatrixEntry

    identity_entry: CoglMatrixEntry

    ### A cache of the last (immutable) matrix stack entries that were flushed to the GL matrix builtins
    ##builtin_flushed_projection: CoglMatrixEntryCache
    ##builtin_flushed_modelview: CoglMatrixEntryCache
    opaque_color_pipeline: Ptr # CoglPipeline (FIXME: no longer present)

    ##texture_units: Ptr # GArray
    ##active_texture_unit: Int
    ##
    ##legacy_fog_state: CoglPipelineFogState
    ##
    ### Pipelines
    ##opaque_color_pipeline: Ptr # CoglPipeline   # used for set_source_color
    ##blended_color_pipeline: Ptr # CoglPipeline   # used for set_source_color
    ##texture_pipeline: Ptr # CoglPipeline   # used for set_source_texture
    codegen_header_buffer: Ptr # GString
    codegen_source_buffer: Ptr # GString
    codegen_boilerplate_buffer: Ptr # GString
    ##source_stack: Ptr # GList
    ##
    ##legacy_state_set: Int

    pipeline_cache: Ptr # CoglPipelineCache

    # Textures
    default_gl_texture_2d_tex: Ptr # CoglTexture2D
    ##default_gl_texture_3d_tex: Ptr # CoglTexture3D
    ##default_gl_texture_rect_tex: Ptr # CoglTextureRectangle

    # Central list of all framebuffers so all journals can be flushed
    # at any time.
    framebuffers: Ptr # GList

    # Global journal buffers
    journal_flush_attributes_array: Ptr # GArray
    journal_clip_bounds: Ptr # GArray

    ##polygon_vertices: Ptr # GArray

    # Some simple caching, to minimize state changes...
    current_pipeline: Ptr # CoglPipeline
    current_pipeline_changes_since_flush: ULong
    current_pipeline_with_color_attrib: CoglBool
    current_pipeline_unknown_color_alpha: CoglBool
    current_pipeline_age: ULong

    gl_blend_enable_cache: CoglBool

    depth_test_enabled_cache: CoglBool
    depth_test_function_cache: CoglDepthTestFunction
    depth_writing_enabled_cache: CoglBool
    depth_range_near_cache: F32
    depth_range_far_cache: F32

    legacy_depth_test_enabled: CoglBool

    current_buffer: Annotated[list[Ptr], FixedSize(COGL_BUFFER_BIND_TARGET_COUNT)] # CoglBuffer

    # Framebuffers
    ##framebuffer_stack: Ptr # GSList
    ##window_buffer: Ptr # CoglFramebuffer
    current_draw_buffer_state_flushed: ULong
    current_draw_buffer_changes: ULong
    current_draw_buffer: Ptr # CoglFramebuffer
    current_read_buffer: Ptr # CoglFramebuffer

    have_last_offscreen_allocate_flags: GBool
    last_offscreen_allocate_flags: CoglOffscreenAllocateFlags

    swap_callback_closures: Ptr # GHashTable
    next_swap_callback_id: Int

    onscreen_events_queue: CoglList
    onscreen_dirty_queue: CoglList
    onscreen_dispatch_idle: Ptr # CoglClosure

    ##current_gles2_context: Ptr # CoglGLES2Context
    ##gles2_context_stack: GQueue

    # This becomes TRUE the first time the context is bound to an
    # onscreen buffer. This is used by cogl-framebuffer-gl to determine
    # when to initialise the glDrawBuffer state
    was_bound_to_onscreen: CoglBool

    # Primitives
    ##current_path: Ptr # CoglPath
    stencil_pipeline: Ptr # CoglPipeline

    ### Pre-generated VBOs containing indices to generate GL_TRIANGLES out of a vertex array of quads
    ##quad_buffer_indices_byte: Ptr # CoglIndices
    ##quad_buffer_indices_len: UInt
    ##quad_buffer_indices: Ptr # CoglIndices

    rectangle_byte_indices: Ptr # CoglIndices
    rectangle_short_indices: Ptr # CoglIndices
    rectangle_short_indices_len: Int

    ##in_begin_gl_block: CoglBool
    ##
    ##texture_download_pipeline: Ptr # CoglPipeline
    blit_texture_pipeline: Ptr # CoglPipeline

    atlases: Ptr # GSList
    atlas_reorganize_callbacks: GHookList

    # This debugging variable is used to pick a colour for visually
    # displaying the quad batches. It needs to be global so that it can
    # be reset by cogl_clear. It needs to be reset to increase the
    # chances of getting the same colour during an animation
    journal_rectangles_color: U8

    # Cached values for GL_MAX_TEXTURE_[IMAGE_]UNITS to avoid calling glGetInteger too often
    max_texture_units: GLint
    max_texture_image_units: GLint
    max_activateable_texture_units: GLint

    # Fragment processing programs
    ##current_program: CoglHandle
    ##
    ##current_fragment_program_type: CoglPipelineProgramType
    ##current_vertex_program_type: CoglPipelineProgramType
    current_gl_program: GLuint

    current_gl_dither_enabled: CoglBool
    ##current_gl_color_mask: CoglColorMask
    current_gl_draw_buffer: GLenum

    # Clipping
    current_clip_stack_valid: CoglBool
    current_clip_stack: Ptr # CoglClipStack
    ##current_clip_stack_uses_stencil: CoglBool

    buffer_map_fallback_array: Ptr # GByteArray
    buffer_map_fallback_in_use: CoglBool
    buffer_map_fallback_offset: Size

    ##rectangle_state: CoglWinsysRectangleState

    sampler_cache: Ptr # CoglSamplerCache

    # things get weird from here, so not transcribed.
    # it begins with an ifdef COGL_HAS_XLIB_SUPPORT

CoglAtlasFlags = Int  # CLEAR_TEXTURE, DISABLE_MIGRATION
CoglAtlasUpdatePositionCallback = Ptr

# based on https://gitlab.gnome.org/GNOME/mutter/-/blob/41.2/cogl/cogl/cogl-pixel-format.h
@enum.unique
class CoglPixelFormat(enum.Enum):
    _ignore_ = 'Flag'
    class Flag(enum.Enum):
        A = enum.auto()
        BGR = enum.auto()
        AFIRST = enum.auto()
        PREMULT = enum.auto()
        DEPTH = enum.auto()
        STENCIL = enum.auto()

    def __new__(cls, bpp, flags):
        obj = object.__new__(cls)
        obj._value_ = bpp, frozenset(flags)
        return obj

    @staticmethod
    def __load__(x: int) -> 'CoglPixelFormat':
        consume = lambda x, n: (x & ~((~0) << n), x >> n)
        x, bpp = consume(x, 4)
        x, rawflags = consume(x, 12)
        x, seq = consume(x, 8)
        assert seq == 0  # no formats currently use seq number

        flags = set()
        for flag in CoglPixelFormat.Flag:
            rawflags, bit = consume(rawflags, 1)
            if bit: flags.add(flag)
        assert not rawflags

        return CoglPixelFormat((bpp, flags))

    ANY           = 0, {}
    A_8           = 1, {Flag.A}

    RGB_565       = 4, {}
    RGBA_4444     = 5, {Flag.A}
    RGBA_5551     = 6, {Flag.A}
    YUV           = 7, {}
    G_8           = 8, {}

    RG_88         = 9, {}

    RGB_888       = 2, {}
    BGR_888       = 2, {Flag.BGR}

    RGBA_8888     = 3, {Flag.A}
    BGRA_8888     = 3, {Flag.A, Flag.BGR}
    ARGB_8888     = 3, {Flag.A, Flag.AFIRST}
    ABGR_8888     = 3, {Flag.A, Flag.BGR, Flag.AFIRST}

    RGBA_1010102  = 13, {Flag.A}
    BGRA_1010102  = 13, {Flag.A, Flag.BGR}
    XRGB_2101010  = 13, {Flag.AFIRST}
    ARGB_2101010  = 13, {Flag.A, Flag.AFIRST}
    XBGR_2101010  = 13, {Flag.BGR, Flag.AFIRST}
    ABGR_2101010  = 13, {Flag.A, Flag.BGR, Flag.AFIRST}

    RGBA_FP_16161616 = 11, {Flag.A}
    BGRA_FP_16161616 = 11, {Flag.A, Flag.BGR}
    XRGB_FP_16161616 = 11, {Flag.AFIRST}
    ARGB_FP_16161616 = 11, {Flag.A, Flag.AFIRST}
    XBGR_FP_16161616 = 11, {Flag.BGR, Flag.AFIRST}
    ABGR_FP_16161616 = 11, {Flag.A, Flag.BGR, Flag.AFIRST}

    RGBA_8888_PRE = 3, {Flag.A, Flag.PREMULT}
    BGRA_8888_PRE = 3, {Flag.A, Flag.PREMULT, Flag.BGR}
    ARGB_8888_PRE = 3, {Flag.A, Flag.PREMULT, Flag.AFIRST}
    ABGR_8888_PRE = 3, {Flag.A, Flag.PREMULT, Flag.BGR, Flag.AFIRST}
    RGBA_4444_PRE = 5, {Flag.A, Flag.PREMULT}
    RGBA_5551_PRE = 6, {Flag.A, Flag.PREMULT}

    RGBA_1010102_PRE = 13, {Flag.A, Flag.PREMULT}
    BGRA_1010102_PRE = 13, {Flag.A, Flag.BGR, Flag.PREMULT}
    ARGB_2101010_PRE = 13, {Flag.A, Flag.AFIRST, Flag.PREMULT}
    ABGR_2101010_PRE = 13, {Flag.A, Flag.BGR, Flag.AFIRST, Flag.PREMULT}

    RGBA_FP_16161616_PRE = 11, {Flag.A, Flag.PREMULT}
    BGRA_FP_16161616_PRE = 11, {Flag.A, Flag.PREMULT, Flag.BGR}
    ARGB_FP_16161616_PRE = 11, {Flag.A, Flag.PREMULT, Flag.AFIRST}
    ABGR_FP_16161616_PRE = 11, {Flag.A, Flag.PREMULT, Flag.BGR, Flag.AFIRST}

    DEPTH_16  = 9, {Flag.DEPTH}
    DEPTH_32  = 3, {Flag.DEPTH}

    DEPTH_24_STENCIL_8 = 3, {Flag.DEPTH, Flag.STENCIL}

@enum.unique
class CoglTextureComponents(enum.Enum):
    COGL_TEXTURE_COMPONENTS_A = 1
    COGL_TEXTURE_COMPONENTS_RG = 2
    COGL_TEXTURE_COMPONENTS_RGB = 3
    COGL_TEXTURE_COMPONENTS_RGBA = 4
    COGL_TEXTURE_COMPONENTS_DEPTH = 5

@dataclass
class CoglTexture(Struct):
    parent_instance: CoglObject
    context: Ptr # CoglContext
    loader: Ptr # CoglTextureLoader
    framebuffers: Ptr # GList
    max_level_set: Int
    max_level_requested: Int
    width: Int
    height: Int
    allocated: GBool

    # Internal format
    components: Int # FIXME: CoglTextureComponents
    flags: U8 # bit 0: premultiplied

    vtable: Ptr # const CoglTextureVtable

@dataclass
class CoglAtlas(Struct):
    parent_instance: CoglObject

    map: Ptr # CoglRectangleMap

    texture: Ptr # CoglTexture
    texture_format: UInt # FIXME: CoglPixelFormat
    flags: CoglAtlasFlags

    update_position_cb: CoglAtlasUpdatePositionCallback

    pre_reorganize_callbacks: GHookList
    post_reorganize_callbacks: GHookList

@enum.unique
class CoglRectangleMapNodeType(enum.Enum):
    BRANCH = 0
    FILLED_LEAF = 1
    EMPTY_LEAF = 2

@dataclass
class CoglRectangleMapEntry(Struct):
    x: UInt
    y: UInt
    width: UInt
    height: UInt

    @property
    def end_x(self) -> int:
        return self.x + self.width
    @property
    def end_y(self) -> int:
        return self.y + self.height
    @property
    def area(self) -> int:
        return self.width * self.height
    @property
    def valid(self) -> bool:
        return self.end_x <= (1 << 32) \
            and self.end_y <= (1 << 32) \
            and self.width > 0 and self.height > 0

    def contains(self, pos: tuple[float, float]) -> bool:
        return self.x < pos[0] < self.end_x and self.y < pos[1] < self.end_y

    def inside(self, other: 'CoglRectangleMapEntry') -> bool:
        return self.x >= other.x and self.end_x <= other.end_x \
            and self.y >= other.y and self.end_y <= other.end_y

    def non_overlapping(self, other: 'CoglRectangleMapEntry') -> bool:
        return self.end_x <= other.x or other.end_x <= self.x \
            or self.end_y <= other.y or other.end_y <= self.y

class ParsedRectangleMapNode(NamedTuple):
    rect: CoglRectangleMapEntry
    largest_gap: UInt
    child: Union[Optional[Ptr], tuple['ParsedRectangleMapNode', 'ParsedRectangleMapNode']]

    def lookup(self, pos: tuple[float, float]) -> Optional['ParsedRectangleMapNode']:
        if not self.rect.contains(pos): return
        child = self.child
        sublookup = (child[0].lookup(pos) or child[1].lookup(pos)) if type(child) is tuple else None
        return sublookup or self

    def iter_leaves(self) -> Iterator[tuple[CoglRectangleMapEntry, Optional[Ptr]]]:
        if type(self.child) is tuple:
            for child in self.child:
                yield from child.iter_leaves()
        else:
            yield (self.rect, cast(Optional[Ptr], self.child))

@dataclass
class CoglRectangleMapNode(Struct):
    type: Int # FIXME: CoglRectangleMapNodeType
    rectangle: CoglRectangleMapEntry
    largest_gap: UInt
    parent: Ptr # CoglRectangleMapNode

    # branch: (left, right). filled leaf: (data, unused)
    ptr1: Ptr
    ptr2: Ptr

@dataclass
class CoglRectangleMap(Struct):
    root: Ptr # CoglRectangleMapNode

    n_rectangles: UInt

    space_remaining: UInt

    value_destroy_func: GDestroyNotify

    # Stack used for walking the structure. This is only used during
    # the lifetime of a single function call but it is kept here as an
    # optimisation to avoid reallocating it every time it is needed
    stack: Ptr # GArray

    def read(self, st: BinaryIO, parent_rect: Optional[CoglRectangleMapEntry]=None, parent: int=0) -> ParsedRectangleMapNode:
        n_rectangles = 0

        def parse_node(addr: int, parent_rect: Optional[CoglRectangleMapEntry], parent: int) -> ParsedRectangleMapNode:
            st.seek(addr)
            node = CoglRectangleMapNode.__load__(st)
            kind = CoglRectangleMapNodeType(node.type)
            assert node.parent == parent
            assert node.rectangle.valid
            if parent_rect != None:
                assert node.rectangle.inside(parent_rect)
            if kind == CoglRectangleMapNodeType.BRANCH:
                left, right = (parse_node(p, node.rectangle, addr) for p in (node.ptr1, node.ptr2))
                assert left.rect.non_overlapping(right.rect)
                child = (left, right)
            else:
                if filled := (kind == CoglRectangleMapNodeType.FILLED_LEAF):
                    assert node.ptr1
                    nonlocal n_rectangles
                    n_rectangles += 1
                child = node.ptr1 if filled else None
            return ParsedRectangleMapNode(node.rectangle, node.largest_gap, child)

        node = parse_node(self.root, parent_rect, parent)
        assert n_rectangles == self.n_rectangles
        return node

@dataclass
class CoglAtlasTexture(Struct):
    parent_instance: CoglTexture

    # The format that the texture is in. This isn't necessarily the
    # same format as the atlas texture because we can store
    # pre-multiplied and non-pre-multiplied textures together
    internal_format: UInt # CoglPixelFormat

    # The rectangle that was used to add this texture to the
    # atlas. This includes the 1-pixel border
    rectangle: CoglRectangleMapEntry

    # The atlas that this texture is in. If the texture is no longer in
    # an atlas then this will be NULL. A reference is taken on the
    # atlas by the texture (but not vice versa so there is no cycle)
    atlas: Ptr # CoglAtlas

    # Either a CoglSubTexture representing the atlas region for easy
    # rendering or if the texture has been migrated out of the atlas it
    # may be some other texture type such as CoglTexture2D
    sub_texture: Ptr # CoglTexture


# COGL-PANGO

@dataclass
class CoglPangoFontMap(Struct):
    parent_instance: PangoCairoFontMap

    @dataclass
    class Priv(Struct):
        ctx: Ptr # CoglContext
        renderer: Ptr # PangoRenderer

    def get_priv(self, st: BinaryIO) -> Priv:
        # read priv_key quark
        st.seek(COGL_PANGO_FONT_MAP__PRIV_KEY_ADDR)
        priv_key, = unpack('I', st.read(4))

        st.seek(self.parent_instance.get_qdata(st, priv_key))
        return CoglPangoFontMap.Priv.__load__(st)

@dataclass
class CoglPangoRendererCaches(Struct):
    glyph_cache: Ptr # CoglPangoGlyphCache
    pipeline_cache: Ptr # CoglPangoPipelineCache

@dataclass
class CoglPangoRenderer(Struct):
    parent_instance: PangoRenderer

    ctx: Ptr # CoglContext

    # Two caches of glyphs as textures and their corresponding pipeline
    # caches, one with mipmapped textures and one without
    no_mipmap_caches: CoglPangoRendererCaches
    mipmap_caches: CoglPangoRendererCaches

    use_mipmapping: CoglBool

    # The current display list that is being built
    display_list: Ptr # CoglPangoDisplayList

    @property
    def active_caches(self) -> CoglPangoRendererCaches:
        return self.mipmap_caches if self.use_mipmapping else self.no_mipmap_caches

@dataclass
class CoglPangoGlyphCache(Struct):
    ctx: Ptr # CoglContext

    # Hash table to quickly check whether a particular glyph in a
    # particular font is already cached
    hash_table: Ptr # GHashTable

    # List of CoglAtlases
    atlases: Ptr # GSList

    # List of callbacks to invoke when an atlas is reorganized
    reorganize_callbacks: GHookList

    # TRUE if we've ever stored a texture in the global atlas. This is
    # used to make sure we only register one callback to listen for
    # global atlas reorganizations
    using_global_atlas: CoglBool

    # True if some of the glyphs are dirty. This is used as an
    # optimization in _cogl_pango_glyph_cache_set_dirty_glyphs to avoid
    # iterating the hash table if we know none of them are dirty
    has_dirty_glyphs: CoglBool

    # Whether mipmapping is being used for this cache. This only
    # affects whether we decide to put the glyph in the global atlas
    use_mipmapping: CoglBool

@dataclass(frozen=True)
class CoglPangoGlyphCacheKey(Struct):
    font: Ptr # PangoFont
    glyph: PangoGlyph

@dataclass
class CoglPangoGlyphCacheValue(Struct):
    texture: Ptr # CoglTexture

    tx1: F32
    ty1: F32
    tx2: F32
    ty2: F32

    tx_pixel: Int
    ty_pixel: Int

    draw_x: Int
    draw_y: Int
    draw_width: Int
    draw_height: Int

    # flags:
    #  bit 0: dirty:
    #   This will be set to TRUE when the glyph atlas is reorganized
    #   which means the glyph will need to be redrawn
    #  bit 1: has_color
    #   Set to TRUE if the glyph has colors (eg. emoji)
    flags: U8


# CLUTTER

@dataclass
class ClutterImage(Struct):
    parent_instance: GObject
    priv: Ptr # ClutterImagePrivate

@dataclass
class ClutterImagePrivate(Struct):
    texture: Ptr # CoglTexture


# ST

@dataclass
class StTextureCache(Struct):
    parent_instance: GObject
    priv: Ptr # StTextureCachePrivate

@dataclass
class StTextureCachePrivate(Struct):
    icon_theme: Ptr # GtkIconTheme
    settings: Ptr # GSettings

    # Things that were loaded with a cache policy != NONE
    keyed_cache: Ptr # GHashTable   # char* -> ClutterImage*
    keyed_surface_cache: Ptr # GHashTable   # char* -> cairo_surface_t*

    used_scales: Ptr # GHashTable   # Set: double

    # Presently this is used to de-duplicate requests for GIcons and async URIs.
    outstanding_requests: Ptr # GHashTable   # char* -> AsyncTextureLoadData*

    # File monitors to evict cache data on changes
    file_monitors: Ptr # GHashTable   # char* -> GFileMonitor*

    cancellable: Ptr # GCancellable


# MAIN LOGIC

# access the process' memory as a readable stream `st`,
# and get info about its mapped files so we can know where they are loaded
if isinstance(PROC_SRC, int):
    # live process: use /proc/pid/...
    st = open(f'/proc/{PROC_SRC}/mem', 'rb')
    with open(f'/proc/{PROC_SRC}/maps', 'rb') as f:
        mappings = list(map(ProcfsMapEntry.parse, f))
    get_base_addr = lambda fname: next( mm.start - mm.pgoff for mm in mappings if mm.filename == fname )
else:
    # corefile: to fetch this info from the corefile, at the
    # moment I use emucore. FIXME: it's a very heavy dependency,
    # designed to do much more than this. I should ideally replace it.
    from emucore import EmuCore
    emu = EmuCore(PROC_SRC, mapping_load_kwargs=dict(blacklist={ '/' }))
    st = cast(BinaryIO, emu.mem())
    get_base_addr = lambda fname: next( vma.start - vma.offset for fname2, vma in emu.mappings if fname == fname2 )


# GLOBAL VARIABLES. these are addresses of a symbol in some library,
# and are here hardcoded to a certain address that I've looked up myself
# because I have no symbols.
# ideally (if I had symbols) I'd perform a symbol lookup instead of hardcoding
COGL_PANGO_FONT_MAP__PRIV_KEY_ADDR = \
    get_base_addr(b'/usr/lib/mutter-9/libmutter-cogl-pango-9.so.0.0.0') + 0xb038
ST_TEXTURE_CACHE__INSTANCE_ADDR = \
    get_base_addr(b'/usr/lib/gnome-shell/libst-1.0.so') + 0x89700

pp = PrettyPrinter(indent=4)

S = TypeVar('S', bound=FixedSerializable)
def load(cls: type[S], n: int) -> S:
    st.seek(n)
    return cls.__load__(st)

# start by reading global state (FontMap, Context, Renderer, GlyphCache)

font_map = load(CoglPangoFontMap, FONT_MAP)
font_map_priv = font_map.get_priv(st)

ctx = load(CoglContext, font_map_priv.ctx)

renderer = load(CoglPangoRenderer, font_map_priv.renderer)
assert renderer.ctx == font_map_priv.ctx

glyph_cache = load(CoglPangoGlyphCache, renderer.active_caches.glyph_cache)
assert glyph_cache.using_global_atlas and not glyph_cache.atlases
raw_glyphs = load(GHashTable, glyph_cache.hash_table).read(st)
assert len(raw_glyphs) == len(set(raw_glyphs.values()))
glyphs = { load(CoglPangoGlyphCacheKey, k): load(CoglPangoGlyphCacheValue, v) for k, v in raw_glyphs.items() }
assert len(raw_glyphs) == len(glyphs)

glyphs_without_texture = { k for k, v in glyphs.items() if not v.texture }
print(f'{len(glyphs_without_texture)} / {len(glyphs)} glyphs without texture')
rev_glyphs = { v.texture: (k, v) for k, v in glyphs.items() if v.texture }
assert len(rev_glyphs) == len(glyphs) - len(glyphs_without_texture)
for k, v in rev_glyphs.values():
    assert v.tx1 == 0 and v.ty1 == 0
    assert v.tx2 == 1 and v.ty2 == 1
    assert v.tx_pixel==0 and v.ty_pixel==0

# pp.pprint(ctx)
# print()
# pp.pprint(renderer)
# print()
# pp.pprint(glyph_cache)
# pp.pprint(glyphs)

# read texture cache (not really relevant to this bug, but useful nonetheless)

st.seek(ST_TEXTURE_CACHE__INSTANCE_ADDR)
texture_cache = unpack('Q', st.read(8))[0]
texture_cache = load(StTextureCache, texture_cache)
texture_cache = load(StTextureCachePrivate, texture_cache.priv)
keyed_cache = load(GHashTable, texture_cache.keyed_cache).read(st)
def get_clutter_texture(key: str, addr: int) -> Optional[tuple[Ptr, int]]:
    if key.startswith('st-theme-node-corner:'):  # WTF, these do not point to a GObject...
        print('skipping cache entry:', hex(addr), repr(key))
        return None
    image = load(ClutterImage, addr)
    priv = load(ClutterImagePrivate, image.priv)
    assert priv.texture and image.parent_instance.ref_count > 0
    return priv.texture, image.parent_instance.ref_count - 1
keyed_cache = { read_str(st, k): get_clutter_texture(read_str(st, k), v) for k, v in keyed_cache.items() }
keyed_cache = { k: v for k, v in keyed_cache.items() if v }
rev_keyed_cache = { v: (k, n) for k, (v, n) in keyed_cache.items() }
assert len(keyed_cache) == len(rev_keyed_cache)

# dump global atlases

atlases = list(GSList.read(ctx.atlases, st))
atlas_textures: dict[Ptr, CoglAtlasTexture] = {}

for idx, aaddr in enumerate(atlases):
    # load atlas
    atlas = load(CoglAtlas, aaddr)

    # load backing texture
    texture = load(CoglTexture, atlas.texture)
    width, height = texture.width, texture.height
    area = width * height

    # download backing texture
    def load_texture(fname: str):
        with open(fname, 'rb') as f:
            width, height = unpack('II', f.read(8))
            return np.fromfile(f, dtype='uint8').reshape(height, width, 4)
    if isinstance(PROC_SRC, int):
        # live process: invoke GDB with the 'dumptexture' script to dump it
        run(['gdb', '-p', f'{PROC_SRC}', '-batch-silent', '-ex', f'set $texture={atlas.texture}', '-x', 'dumptexture'], check=True)
        texture = load_texture('/tmp/texturedata')
    else:
        # corefile: use it if saved at the expected filename
        if os.path.exists(fname := PROC_SRC + f'.texture.{atlas.texture}'):
            texture = load_texture(fname)
        else:
            print(f'warning: {repr(fname)} not found, falling back to empty texture')
            texture = np.zeros((height, width, 4), dtype='uint8')
    assert texture.shape == (height, width, 4) and texture.dtype == 'uint8'

    # load rectangle tree
    raw_tree = load(CoglRectangleMap, atlas.map)
    tree = raw_tree.read(st, CoglRectangleMapEntry(0, 0, width, height))

    # load textures pointed to by that tree
    for rect, taddr in tree.iter_leaves():
        if taddr == None: continue
        assert taddr not in atlas_textures
        atlas_textures[taddr] = load(CoglAtlasTexture, taddr)

    # render everything

    print(f'\n\nAtlas {idx}:')
    pp.pprint(atlas)
    pp.pprint(raw_tree)

    usage = sum(rect.area for rect, t in tree.iter_leaves() if t)
    fig, ax = plt.subplots(tight_layout=True)
    ax.set_title(f'Atlas {idx} ({raw_tree.n_rectangles} allocations) ({usage/area*100:.1f}% used)')
    ax.set_facecolor("white")
    ax.imshow(texture)

    for rect, taddr in tree.iter_leaves():
        glyph = icon = texture = trefs = None
        if taddr:
            texture = atlas_textures[taddr]
            trefs = texture.parent_instance.parent_instance.ref_count - 1
            assert trefs >= 0
            assert texture and texture.rectangle == rect and texture.atlas == aaddr
            ptexture = texture.parent_instance
            assert ptexture.width == rect.width - 2 and ptexture.height == rect.height - 2
            # FIXME: assert texture.sub_texture and maybe some fields of parent_instance
            glyph = rev_glyphs.get(taddr)
            if glyph:
                if False: # FIXME
                    print(
                        (rect.x, rect.y),
                        (hex(glyph[0].font), hex(glyph[0].glyph)),
                        glyph[1].flags, (glyph[1].draw_x, glyph[1].draw_y), (glyph[1].draw_width, glyph[1].draw_height)
                    )
                assert rect.width >= 3 and rect.height >= 3  # for me
                assert glyph[1].draw_width == ptexture.width and glyph[1].draw_height == ptexture.height
                #print(glyph[1].flags, (glyph[1].draw_x, glyph[1].draw_y))  # print rest of data in glyph[1]
            icon = rev_keyed_cache.get(taddr)
            assert sum(1 for x in (glyph, icon) if x) <= 1
        opts = dict(color='yellow', alpha=0.4, fill=True) if glyph else \
            dict(color='red', alpha=0.5, fill=True) if icon and not icon[1] and trefs == 0 else \
            dict(color='cyan', alpha=0.4, fill=True) if icon else \
            dict(color='blue', alpha=0.4, fill=True) if texture else \
            dict(color='gray', alpha=0.5, fill=True)
        opts['fill'] = False
        ax.add_patch(Rectangle((rect.x - .5, rect.y - .5), rect.width, rect.height, gid=f'texture_{taddr}', **opts))

    # interactivity

    def describe_allocation(taddr: Ptr) -> str:
        texture = atlas_textures[taddr]
        trefs = texture.parent_instance.parent_instance.ref_count
        ref_desc = f' ({trefs or "no"} refs)' if trefs != 1 else ''
        if (icon := rev_keyed_cache.get(taddr)):
            icon_key, refs = icon
            return f'{repr(icon_key[:50])} ({refs or "no"} image refs)' + ref_desc
        if (glyph := rev_glyphs.get(taddr)):
            gkey, gval = glyph
            desc = f'font {gkey.font:#x}, glyph {gkey.glyph}'
            if (gval.flags >> 0) & 1: desc += ', DIRTY'
            if (gval.flags >> 1) & 1: desc += ', COLOR'
            return desc + ref_desc
        return 'other' + ref_desc

    def onmove(event):
        # ignore events outside the axis, or outside the tree
        pos = (event.xdata, event.ydata)
        if pos[0] is None or pos[1] is None: return
        pos_desc = f'x={pos[0]:.1f}, y={pos[1]:.1f}'
        if not (node := tree.lookup(pos)): return

        # describe mouse position + current tree node
        rect, _, child = node
        rect_desc = f'({rect.x}, {rect.y}) ({rect.width}, {rect.height})'
        child_desc = \
            '[branch]' if type(child) is tuple else \
            '[unallocated]' if not child else \
            describe_allocation(cast(Ptr, child))
        fig.canvas.toolbar.set_message(f'{rect_desc}  {pos_desc}\n{child_desc}')
    fig.canvas.mpl_connect("motion_notify_event", onmove)

# FIXME: verify classes?
assert not (outliers := set(rev_glyphs) - set(atlas_textures))
plt.show()