tracked_class.py

import gdb
from datetime import datetime
import sys

uintptr_t = gdb.lookup_type("uintptr_t")

if (sys.version_info > (3, 0)):
    raw_input = input


def read_command_lines():
    #
    # It is really unfortunate that we there does not appear to be a way to
    # execute the gdb function `read_command_lines` from python.
    #
    ret = []
    gdb.write('End with a line saying just "end".\n')
    line = raw_input(">")
    while line != "end":
        ret.append(line)
        line = raw_input(">")
    return ret


def FunctionForPc(pc):
    orig_pc = pc
    try:
        block = gdb.block_for_pc(int(pc))
        while block:
            if block.function:
                return str(block.function)
            block = block.superblock
    except:
        pass

    #
    # If we couldn't figure it out by looking at the block, it is still
    # possible that gdb can figure out something for it (e.g. if it is
    # in JIT-ed code, etc. If all else fails, return the original address
    # as hex.
    #
    if isinstance(orig_pc, gdb.Value):
        return str(orig_pc)
    else:
        return hex(int(orig_pc))


def TrackedClassWhatis(tc):
    assert isinstance(tc, gdb.Value)

    timeval = tc['ti_metadata']['tim_timestamp']
    seconds = int(timeval['tv_sec'].cast(uintptr_t))
    micros = int(timeval['tv_usec'].cast(uintptr_t))
    alloctime = datetime.utcfromtimestamp(seconds + (1000000.0 / micros))
    gdb.write(str(tc) + " is a " + str(tc.type) +
              " allocated by thread " +
              str(tc['ti_metadata']['tim_thread']) +
              " at " + str(alloctime) +
              " from:\n")
    depth = int(tc['ti_metadata']['tim_stack_depth'])
    raw_stack = (tc['ti_metadata']['tim_stack'][i] for i in range(depth))
    for pc in raw_stack:
        gdb.write("\t" + FunctionForPc(pc) + "\n")
    gdb.write("\n")


def TrackedClassWalk(class_name, filter=lambda _: True):
    instance = gdb.parse_and_eval(
        'TrackedClass<%s>::s_tracked_instances' % class_name)
    while instance:
        gdb.execute("set $_ = ('%s' *)%s" % (class_name, str(instance)))
        gdb.execute("set $__ = *$_")
        if filter(instance):
            yield instance
        instance = instance['ti_next_instance']


class TrackedClassCommand(gdb.Command):
    """Operations on tracked clases."""
    def __init__(self):
        super(TrackedClassCommand, self).__init__(
            "tracked-class", gdb.COMMAND_DATA, gdb.COMPLETE_COMMAND, True)
TrackedClassCommand()


class TrackedClassWhatisCommand(gdb.Command):
    """Print allocation metadata for a TrackedClass."""

    def __init__(self):
        super(TrackedClassWhatisCommand, self).__init__(
            "tracked-class whatis", gdb.COMMAND_DATA, gdb.COMPLETE_EXPRESSION)

    def invoke(self, argument, from_tty):
        tc = gdb.parse_and_eval(argument)
        TrackedClassWhatis(tc)
        self.dont_repeat()
TrackedClassWhatisCommand()


class TrackedClassWalkCommand(gdb.Command):
    """Print all instances of a TrackedClass.

    If an optional filter is used, $_ and $__ can be used in the filter
    expression to reference the instance.

    Usage:
        tracked-class walk <type> [if <filter>]

    Example:
        (gdb) tracked-class walk LLVMEngine::CompiledUnit if $_->m_loaded
    """

    def __init__(self):
        super(TrackedClassWalkCommand, self).__init__(
            "tracked-class walk", gdb.COMMAND_DATA, gdb.COMPLETE_EXPRESSION)

    def invoke(self, argument, from_tty):
        args = gdb.string_to_argv(argument)
        if (len(args) == 0 or len(args) == 2 or
                (len(args) > 1 and args[1] != "if")):
            raise gdb.GdbError("Usage: <type> [if <filter expression>]")

        _filter = lambda _: True
        if len(args) > 1:
            _filter = lambda _: bool(gdb.parse_and_eval(" ".join(args[2:])))

        for tc in TrackedClassWalk(args[0], _filter):
            gdb.write(str(tc) + "\n")
        self.dont_repeat()
TrackedClassWalkCommand()


class TrackedClassWalkCommandsCommand(gdb.Command):
    """Execute the given commands on all instances of a TrackedClass.

    The commands can reference the instance via $_ and $__.

    If an optional filter is used, $_ and $__ can be used in the filter
    expression to reference the instance.

    Usage:
        tracked-class walk-commands <type> [if <filter>]

    Example:
        (gdb) tracked-class walk-commands Mbc::Module if $_->m_loaded
        Type commands for each instance.
        End with a line saying just "end".
        >tracked-class whatis $_
        >end
    """

    def __init__(self):
        super(TrackedClassWalkCommandsCommand, self).__init__(
            "tracked-class walk-commands", gdb.COMMAND_DATA,
            gdb.COMPLETE_EXPRESSION)

    def invoke(self, argument, from_tty):
        args = gdb.string_to_argv(argument)
        if (len(args) == 0 or len(args) == 2 or
                (len(args) > 1 and args[1] != "if")):
            raise gdb.GdbError("Usage: <type> [if <filter expression>]")

        _filter = lambda _: True
        if len(args) > 1:
            _filter = lambda _: bool(gdb.parse_and_eval(" ".join(args[2:])))

        gdb.write("Type commands for each instance.\n")
        commands = read_command_lines()
        for _ in TrackedClassWalk(args[0], _filter):
            for command in commands:
                gdb.execute(command)
        self.dont_repeat()
TrackedClassWalkCommandsCommand()

TrackedClass.h

#pragma once

// A parent class using CRTP to track all instances of the derived class.
//
// This supports two python functions for use in gdb:
//
//   - TrackedClassWalk(derived_class_name, filter=lambda _: True):
//
//       A generator that iterates over all instances of the derived class
//       in existance. This currently relies on embedding additional data in
//       the Derived class to track all instances in a doubly linked list
//       and consequently is only enabled in debug builds. Future versions
//       of TrackedClass will instead use memory allocator metadata to enable
//       them to support TrackedClassWalk with no overhead in release builds
//       as well as debug builds.
//
//   - TrackedClassWhatis(gdb_value, pretty=True):
//
//       Prints to the gdb console metadata about the value, including
//       the stacktrace where it was allocated. This currently requires
//       embedding the stack trace in the Derived class and consequently
//       is only enabled in debug builds. Future versions of TrackedClass
//       will instead use memory allocator metadata to avoid the need to
//       embed data in the Derived class, but will still only support
//       TrackedClassWhatis in debug builds.
//
// == Usage ==
//
//     (gdb) source tracked_class.py
//     (gdb) python print(len(list(TrackedClassWalk('LLVMEngine::CompiledUnit'))))
//     23
//     (gdb) python le_cu = next(TrackedClassWalk('LLVMEngine::CompiledUnit'))
//     (gdb) python print le_cu
//     0x5f4db00
//     (gdb) python m_u = next(TrackedClassWalk('Mbc::Unit', lambda u: u['m_cu']['_M_t']['_M_head_impl'] == le_cu))
//     (gdb) python print m_u
//     0x5c776e0
//     (gdb) python TrackedClassWhatis(le_cu)
//     0x64d77f0 is a LLVMEngine::CompiledUnit * allocated by thread {_M_thread = 140572196010240}
//         TrackedClass<LLVMEngine::CompiledUnit>::TrackedClass()
//         LLVMEngine::CompiledUnit::CompiledUnit(bool)
//         LLVMEngine::CompiledUnitBuilder::Create(bool, bool)
//         LLVMEngine::Compile(Mbc::Unit*, char const*, bool, bool)
//         Mbc::Unit::getFunction<void, void*>(std::string const&, std::function<void (void*)>&, InterpreterMode)
//         GetCreateIndexesFn(AllMetadataTable const&, Mbc::Unit*, unsigned long, InterpreterMode, AlterTableInfo*)
//         CreateTableEntry(DynamicObject&, char const*, int, std::unique_ptr<AllMetadataTable, std::default_delete<AllMetadataTable> >&, unsigned long, DatabaseMetadataManager*, char const*, AlterTableInfo*, OperatorSelect*, OperatorSelect*, bool)
//         Metametadata::CreateInternalTable(int, INTERNAL_TABLE_TYPE, DatabasesListEntry*)
//         DatabaseMetadataManager::DatabaseMetadataManager(int, MetadataDatabase*, bool, bool, bool, size_t, DatabasesListEntry*)
//         DatabasesListEntry::DatabasesListEntry(int, MetadataDatabase*, bool, bool, bool, size_t)
//         MetadataManager::CreateDatabase(int, char const*, bool, bool, bool, unsigned long, unsigned int, bool, bool, MetadataReplica*)
//         InitInternalDatabase(int, char const*, bool)
//         MemSqlInitializeMetadataMgr()
//         main(int, char**)
//         __libc_start_main
//         [unknown]
//
//
#ifdef NDEBUG
template <typename T> class TrackedClass {};
#else
#include <assert.h>
#include <execinfo.h>
#include <sys/time.h>

#include <mutex>
#include <thread>

template <typename T>
class TrackedClass
{
protected:
    __attribute__((noinline))
    ~TrackedClass()
    {
        assert(ti_magic == TC_ALLOCED_MAGIC);
        ti_magic = TC_FREED_MAGIC;
        std::lock_guard<std::mutex> autolock(T::s_tracked_instances_lock);
        if (ti_prev_instance == nullptr)
        {
            assert(T::s_tracked_instances == static_cast<T *>(this));
            T::s_tracked_instances = ti_next_instance;
        }
        else
        {
            assert(ti_prev_instance->ti_next_instance ==
                static_cast<T *>(this));
            ti_prev_instance->ti_next_instance = ti_next_instance;
        }
        if (ti_next_instance != nullptr)
        {
            assert(ti_next_instance->ti_prev_instance ==
                static_cast<T *>(this));
            ti_next_instance->ti_prev_instance = ti_prev_instance;
        }
        T::s_tracked_instances_count--;
    }

    TrackedClass(const TrackedClass&) = delete;
    TrackedClass(TrackedClass&&) = delete;

    __attribute__((noinline))
    TrackedClass() : ti_magic(TC_ALLOCED_MAGIC),
        ti_next_instance(nullptr), ti_prev_instance(nullptr)
    {
        trackInstance();
    }
private:
    void updateMetadata()
    {
        gettimeofday(&ti_metadata.tim_timestamp, NULL);
        ti_metadata.tim_thread = std::this_thread::get_id();

        void *stack[MAX_STACK_DEPTH + SKIP_STACK_FRAMES];
        ti_metadata.tim_stack_depth =
            backtrace(stack, MAX_STACK_DEPTH + SKIP_STACK_FRAMES);
        assert(ti_metadata.tim_stack_depth > SKIP_STACK_FRAMES);
        ti_metadata.tim_stack_depth -= SKIP_STACK_FRAMES;
        memcpy(ti_metadata.tim_stack, &stack[SKIP_STACK_FRAMES],
            ti_metadata.tim_stack_depth * sizeof(stack[0]));
    }

    void trackInstance()
    {
        assert(ti_prev_instance == nullptr);
        assert(ti_next_instance == nullptr);
        assert(ti_magic == TC_ALLOCED_MAGIC);

        updateMetadata();

        ti_prev_instance = nullptr;
        std::lock_guard<std::mutex> autolock(T::s_tracked_instances_lock);
        ti_next_instance = T::s_tracked_instances;
        T::s_tracked_instances = static_cast<T *>(this);
        if (ti_next_instance != nullptr)
        {
            assert(ti_next_instance->ti_prev_instance == nullptr);
            ti_next_instance->ti_prev_instance = static_cast<T *>(this);
        }
        T::s_tracked_instances_count++;
    }

    // Capture up to 24 frames of the stack trace but exclude the first 3
    // frames: <updateMetadata, trackInstance, TrackInstance::TrackInstance>.
    //
    static constexpr size_t     MAX_STACK_DEPTH     = 24;
    static constexpr size_t     SKIP_STACK_FRAMES   = 3;
    static constexpr uint64_t   TC_ALLOCED_MAGIC    = 0x4141414141414141;
    static constexpr uint64_t   TC_FREED_MAGIC      = 0x4646464646464646;

    // We annotate all of these as having "default" visibility so that there
    // we guarantee there is always ever a single copy of them, even in the
    // case where it is used in a shared object compiled with
    // -fvisibility=hidden.
    //
    // WARNING A dlcose of a library using a TrackedClass should invoke
    // std::mutex::~mutex on the s_tracked_instances_lock if the library
    // had been compiled with -fuse-cxa-atexit. In theory, this should make
    // s_tracked_instances_lock unuseable by other threads; in practice, this
    // appears to work.
    //
    static T            *s_tracked_instances
        __attribute__((visibility("default")));
    static std::mutex   s_tracked_instances_lock
        __attribute__((visibility("default")));
    static uint64_t     s_tracked_instances_count
        __attribute__((visibility("default")));

    uint64_t ti_magic;
    T *ti_next_instance;
    T *ti_prev_instance;

    struct {
        std::thread::id tim_thread;
        struct timeval  tim_timestamp;
        size_t          tim_stack_depth;
        void            *tim_stack[MAX_STACK_DEPTH];
    } ti_metadata;
};

template <typename T> std::mutex TrackedClass<T>::s_tracked_instances_lock;
template <typename T> T *TrackedClass<T>::s_tracked_instances(nullptr);
template <typename T> uint64_t TrackedClass<T>::s_tracked_instances_count(0);

#endif