Babeltrace Comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a8f65923",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Iterable, List, Dict, Any\n",
    "DictEvent = Dict[str, Any]\n",
    "\n",
    "import babeltrace\n",
    "import bt2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "07469f68",
   "metadata": {},
   "outputs": [],
   "source": [
    "trace_dir = '/home/mjcarroll/.ros/profile/config-20230126104756/ros2profile-tracing-session/'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d5b7377",
   "metadata": {},
   "source": [
    "Below is the original implementation from tracetools_analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e72ca56",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_trace_ctf_events(trace_directory: str) -> Iterable[babeltrace.babeltrace.Event]:\n",
    "    \"\"\"\n",
    "    Get the events of a trace.\n",
    "\n",
    "    :param trace_directory: the path to the main/top trace directory\n",
    "    :return: events iterable\n",
    "    \"\"\"\n",
    "    tc = babeltrace.TraceCollection()\n",
    "    tc.add_traces_recursive(trace_directory, 'ctf')\n",
    "    return tc.events\n",
    "\n",
    "\n",
    "def get_trace_events(trace_directory: str) -> List[DictEvent]:\n",
    "    \"\"\"\n",
    "    Get the events of a trace.\n",
    "\n",
    "    :param trace_directory: the path to the main/top trace directory\n",
    "    :return: events\n",
    "    \"\"\"\n",
    "    events: List[DictEvent] = [\n",
    "        event_to_dict(event) for event in get_trace_ctf_events(trace_directory)\n",
    "    ]\n",
    "    return events\n",
    "\n",
    "\n",
    "# List of ignored CTF fields\n",
    "_IGNORED_FIELDS = [\n",
    "    'content_size',\n",
    "    'events_discarded',\n",
    "    'id',\n",
    "    'packet_size',\n",
    "    'packet_seq_num',\n",
    "    'stream_id',\n",
    "    'stream_instance_id',\n",
    "    'timestamp_end',\n",
    "    'timestamp_begin',\n",
    "    'magic',\n",
    "    'uuid',\n",
    "    'v',\n",
    "]\n",
    "_DISCARD = 'events_discarded'\n",
    "\n",
    "\n",
    "def event_to_dict(event: babeltrace.babeltrace.Event) -> DictEvent:\n",
    "    \"\"\"\n",
    "    Convert name, timestamp, and all other keys except those in IGNORED_FIELDS into a dictionary.\n",
    "\n",
    "    :param event: the event to convert\n",
    "    :return: the event as a dictionary\n",
    "    \"\"\"\n",
    "    if hasattr(event, _DISCARD) and event[_DISCARD] > 0:\n",
    "        print(event[_DISCARD])\n",
    "    meta = {'_name': event.name, '_timestamp': event.timestamp}\n",
    "    data = {key: event[key] for key in event.keys() if key not in _IGNORED_FIELDS}\n",
    "    return {**meta, **data}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e0d73b56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 41.5 s, sys: 337 ms, total: 41.8 s\n",
      "Wall time: 41.8 s\n",
      "865725\n"
     ]
    }
   ],
   "source": [
    "%time events = get_trace_events(trace_dir)\n",
    "print(len(events))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1a260b4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_name kmem_mm_page_alloc <class 'str'>\n",
      "_timestamp 1674751677305236696 <class 'int'>\n",
      "gfp_flags 1051840 <class 'int'>\n",
      "migratetype 0 <class 'int'>\n",
      "cpu_id 5 <class 'int'>\n",
      "vpid 816 <class 'int'>\n",
      "order 0 <class 'int'>\n",
      "procname Client manageme <class 'str'>\n",
      "page 18446740140805747072 <class 'int'>\n",
      "vtid 893 <class 'int'>\n",
      "pfn 3320262 <class 'int'>\n"
     ]
    }
   ],
   "source": [
    "event = events[0]\n",
    "for key in event.keys():\n",
    "    print(key, event[key], type(event[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f6f2cded",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.02 s, sys: 356 ms, total: 4.38 s\n",
      "Wall time: 4.38 s\n"
     ]
    }
   ],
   "source": [
    "# Read all of the trace events and just append them to a list\n",
    "\n",
    "def get_trace_events_bt2_simple(trace_directory):\n",
    "    events = []\n",
    "    msg_it = bt2.TraceCollectionMessageIterator(trace_directory)\n",
    "    for msg in msg_it:\n",
    "        if type(msg) is not bt2._EventMessageConst:\n",
    "            continue\n",
    "        events.append(msg)\n",
    "    return events\n",
    "\n",
    "%time events = get_trace_events_bt2_simple(trace_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e7324f84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.2 s, sys: 124 ms, total: 12.4 s\n",
      "Wall time: 12.4 s\n"
     ]
    }
   ],
   "source": [
    "# Read just the trace metadata and put it in a dictionary\n",
    "\n",
    "def get_trace_events_bt2_dict(trace_directory):\n",
    "    msg_it = bt2.TraceCollectionMessageIterator(trace_directory)\n",
    "    events = []\n",
    "\n",
    "    for msg in msg_it:\n",
    "        if type(msg) is not bt2._EventMessageConst:\n",
    "            continue\n",
    "        events.append({\n",
    "            '_name': msg.event.name,\n",
    "            '_timestamp': msg.default_clock_snapshot.ns_from_origin,\n",
    "        })\n",
    "    return events\n",
    "\n",
    "%time events = get_trace_events_bt2_dict(trace_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9020fe2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def event_to_dict_bt2(msg: bt2._EventMessageConst) -> DictEvent:\n",
    "    \"\"\"\n",
    "    Convert name, timestamp, and all other keys except those in IGNORED_FIELDS into a dictionary.\n",
    "\n",
    "    :param event: the event to convert\n",
    "    :return: the event as a dictionary\n",
    "    \"\"\"\n",
    "    ret = {'_name': msg.event.name, '_timestamp': msg.default_clock_snapshot.ns_from_origin}\n",
    "    \n",
    "    def process_field(field):\n",
    "        for (key, value) in field.items():\n",
    "            if key not in _IGNORED_FIELDS:\n",
    "                ret[key] = value\n",
    "    \n",
    "    if msg.event.payload_field:\n",
    "        process_field(msg.event.payload_field)\n",
    "    if msg.event.specific_context_field:\n",
    "        process_field(msg.event.specific_context_field)\n",
    "    if msg.event.common_context_field:\n",
    "        process_field(msg.event.common_context_field)\n",
    "    if msg.event.packet.context_field:\n",
    "        process_field(msg.event.packet.context_field)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1300068d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 8s, sys: 580 ms, total: 1min 9s\n",
      "Wall time: 1min 9s\n"
     ]
    }
   ],
   "source": [
    "# Stuff all available fields in the dictionary, without conversion\n",
    "\n",
    "def get_trace_events_bt2_dict_full(trace_directory):\n",
    "    msg_it = bt2.TraceCollectionMessageIterator(trace_directory)\n",
    "    events = []\n",
    "\n",
    "    for msg in msg_it:\n",
    "        if type(msg) is not bt2._EventMessageConst:\n",
    "            continue\n",
    "        events.append(event_to_dict_bt2(msg))\n",
    "    return events\n",
    "\n",
    "%time events = get_trace_events_bt2_dict_full(trace_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f0b2d729",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_name kmem_mm_page_alloc <class 'str'>\n",
      "_timestamp 1674751677305236720 <class 'int'>\n",
      "page 18446740140805747072 <class 'bt2.field._UnsignedIntegerFieldConst'>\n",
      "pfn 3320262 <class 'bt2.field._UnsignedIntegerFieldConst'>\n",
      "order 0 <class 'bt2.field._UnsignedIntegerFieldConst'>\n",
      "gfp_flags 1051840 <class 'bt2.field._UnsignedIntegerFieldConst'>\n",
      "migratetype 0 <class 'bt2.field._SignedIntegerFieldConst'>\n",
      "vpid 816 <class 'bt2.field._SignedIntegerFieldConst'>\n",
      "procname Client manageme <class 'bt2.field._StringFieldConst'>\n",
      "vtid 893 <class 'bt2.field._SignedIntegerFieldConst'>\n",
      "cpu_id 5 <class 'bt2.field._UnsignedIntegerFieldConst'>\n"
     ]
    }
   ],
   "source": [
    "event = events[0]\n",
    "for key in event.keys():\n",
    "    print(key, event[key], type(event[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1014c59b",
   "metadata": {},
   "outputs": [],
   "source": [
    "BT2_TO_POD_CONVERSIONS = {\n",
    "    bt2._UnsignedIntegerFieldConst: int,\n",
    "    bt2._SignedIntegerFieldConst: int,\n",
    "    bt2._StringFieldConst: str,\n",
    "    bt2._DynamicArrayFieldConst: lambda x: map(int, x),\n",
    "    bt2._StaticArrayFieldConst: lambda x: map(int, x),\n",
    "}\n",
    "\n",
    "\n",
    "def event_to_dict_bt2_convert(msg: bt2._EventMessageConst) -> DictEvent:\n",
    "    \"\"\"\n",
    "    Convert name, timestamp, and all other keys except those in IGNORED_FIELDS into a dictionary.\n",
    "\n",
    "    :param event: the event to convert\n",
    "    :return: the event as a dictionary\n",
    "    \"\"\"\n",
    "    ret = {'_name': msg.event.name, '_timestamp': msg.default_clock_snapshot.ns_from_origin}\n",
    "    \n",
    "    def process_field(field):\n",
    "        for (key, value) in field.items():\n",
    "            if key not in _IGNORED_FIELDS:\n",
    "                ret[key] = BT2_TO_POD_CONVERSIONS[type(value)](value)\n",
    "    \n",
    "    if msg.event.payload_field:\n",
    "        process_field(msg.event.payload_field)\n",
    "    if msg.event.specific_context_field:\n",
    "        process_field(msg.event.specific_context_field)\n",
    "    if msg.event.common_context_field:\n",
    "        process_field(msg.event.common_context_field)\n",
    "    if msg.event.packet.context_field:\n",
    "        process_field(msg.event.packet.context_field)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "51a894a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 7s, sys: 428 ms, total: 1min 7s\n",
      "Wall time: 1min 7s\n"
     ]
    }
   ],
   "source": [
    "# Stuff all available fields in the dictionary, WITH conversion\n",
    "\n",
    "def get_trace_events_bt2_dict_full_convert(trace_directory):\n",
    "    msg_it = bt2.TraceCollectionMessageIterator(trace_directory)\n",
    "    events = []\n",
    "\n",
    "    for msg in msg_it:\n",
    "        if type(msg) is not bt2._EventMessageConst:\n",
    "            continue\n",
    "        events.append(event_to_dict_bt2_convert(msg))\n",
    "    return events\n",
    "\n",
    "%time events = get_trace_events_bt2_dict_full_convert(trace_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e2e6d7d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_name kmem_mm_page_alloc <class 'str'>\n",
      "_timestamp 1674751677305236720 <class 'int'>\n",
      "page 18446740140805747072 <class 'int'>\n",
      "pfn 3320262 <class 'int'>\n",
      "order 0 <class 'int'>\n",
      "gfp_flags 1051840 <class 'int'>\n",
      "migratetype 0 <class 'int'>\n",
      "vpid 816 <class 'int'>\n",
      "procname Client manageme <class 'str'>\n",
      "vtid 893 <class 'int'>\n",
      "cpu_id 5 <class 'int'>\n"
     ]
    }
   ],
   "source": [
    "event = events[0]\n",
    "for key in event.keys():\n",
    "    print(key, event[key], type(event[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db77b9ab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}