txq.bt - monitor the length of an AWS ENA device's transmit queue

txq.bt

#include "txq.h"

kprobe:ena_com_prepare_tx
{
  // grab/cast the transmit queue param
  $io_sq = (struct ena_com_io_sq *) arg0;

  // logic copied from from ena_com_free_q_entries
  $queuelen = (uint16) $io_sq->tail - (uint16) $io_sq->next_to_comp;

  // Create linear histogram of length of the transmit queue for each pid/queue id combo
  @txq[pid, $io_sq->idx] = lhist((uint16) $queuelen, 0, 65, 8);
}

txq.h

// Had to copy all these dependent stucts so I could define ena_com_io_sq

struct ena_com_io_desc_addr {
        u8 __iomem *pbuf_dev_addr; /* LLQ address */
        u8 *virt_addr;
        dma_addr_t phys_addr;
};

struct ena_com_tx_meta {
        u16 mss;
        u16 l3_hdr_len;
        u16 l3_hdr_offset;
        u16 l4_hdr_len; /* In words */
};

struct ena_com_llq_info {
        u16 header_location_ctrl;
        u16 desc_stride_ctrl;
        u16 desc_list_entry_size_ctrl;
        u16 desc_list_entry_size;
        u16 descs_num_before_header;
        u16 descs_per_entry;
        u16 max_entries_in_tx_burst;
        bool disable_meta_caching;
};

struct ena_com_llq_pkt_ctrl {
        u8 *curr_bounce_buf;
        u16 idx;
        u16 descs_left_in_line;
};

struct ena_com_io_bounce_buffer_control {
        u8 *base_buffer;
        u16 next_to_use;
        u16 buffer_size;
        u16 buffers_num;  /* Must be a power of 2 */
};

enum queue_direction {
        ENA_COM_IO_QUEUE_DIRECTION_TX,
        ENA_COM_IO_QUEUE_DIRECTION_RX
};

enum ena_admin_placement_policy_type {
        ENA_ADMIN_PLACEMENT_POLICY_HOST = 1,
        ENA_ADMIN_PLACEMENT_POLICY_DEV  = 3,
};

struct ena_com_io_sq {
        struct ena_com_io_desc_addr desc_addr;
        void *bus;

        u32 __iomem *db_addr;
        u8 __iomem *header_addr;

        enum queue_direction direction;
        enum ena_admin_placement_policy_type mem_queue_type;

        bool disable_meta_caching;

        u32 msix_vector;
        struct ena_com_tx_meta cached_tx_meta;
        struct ena_com_llq_info llq_info;
        struct ena_com_llq_pkt_ctrl llq_buf_ctrl;
        struct ena_com_io_bounce_buffer_control bounce_buf_ctrl;

        u16 q_depth;
        u16 qid;

        u16 idx;
        u16 tail;
        u16 next_to_comp;
        u16 llq_last_copy_tail;
        u32 tx_max_header_size;
        u8 phase;
        u8 desc_entry_size;
        u8 dma_addr_bits;
        u16 entries_in_tx_burst_left;
};