Stop and start `perf` measurements with a named pipe

CMakeLists.txt

cmake_minimum_required(VERSION "3.23")

# C project
project(xenodochial_sinoussi C)

add_executable(prog prog.c)

perf-record-with-fd.log

# To display the perf.data header info, please use --header/--header-only options.
#
#
# Total Lost Samples: 0
#
# Samples: 7K of event 'cycles'
# Event count (approx.): 6271934034
#
# Overhead  Command  Shared Object      Symbol                        
# ........  .......  .................  ..............................
#
   100.00%  prog     prog               [.] _Z3fibm
     0.00%  prog     [kernel.kallsyms]  [k] perf_event_update_userpage
     0.00%  prog     [kernel.kallsyms]  [k] native_flush_tlb_one_user
     0.00%  prog     [kernel.kallsyms]  [k] native_write_msr


#
# (Tip: For a higher level overview, try: perf report --sort comm,dso)
#

perf-stat-with-fd.log

# started on Fri Nov 11 13:50:59 2022


 Performance counter stats for 'build/prog':

          1,966.00 msec task-clock                       #    0.495 CPUs utilized          
                 4      context-switches                 #    2.035 /sec                   
                 0      cpu-migrations                   #    0.000 /sec                   
                 0      page-faults                      #    0.000 /sec                   
     6,263,462,414      cycles                           #    3.186 GHz                    
    22,618,528,756      instructions                     #    3.61  insn per cycle         
     4,023,384,986      branches                         #    2.046 G/sec                  
        14,196,958      branch-misses                    #    0.35% of all branches        

       3.968515104 seconds time elapsed

       0.000000000 seconds user
       0.000000000 seconds sys

prog.c

#include <assert.h>    // assert
#include <stddef.h>    // size_t
#include <stdio.h>     // printf
#include <stdlib.h>    // atoi, getenv
#include <string.h>    // strcmp
#include <unistd.h>    // read, write

size_t fib(size_t n)
{
    if (n == 0)
        return 0;
    else if (n == 1)
        return 1;
    else
        return fib(n - 1) + fib(n - 2);
}

int main(int argc, char* argv[])
{
    int perf_ctl_fd;
    int perf_ack_fd;
    char ack[5];
    size_t n;
    size_t r;

    // // Make sure we have the right number of arguments
    // if (argc != 3)
    // {
    //     fprintf(stderr, "Received %d arguments, expected 2.\n", argc - 1);
    //     fprintf(stderr, "Usage: %s <fd1> <fd2>", argv[0]);
    //     return 1;
    // }
    // perf_ctl_fd = atoi(argv[1]);
    // perf_ack_fd = atoi(argv[2]);

    perf_ctl_fd = atoi(getenv("PERF_CTL_FD"));
    perf_ack_fd = atoi(getenv("PERF_ACK_FD"));

    // Sleep for 2 seconds
    sleep(2);

    // scanf("%zu", &n);
    n = 45;

    // Start the performance counter and read the ack
    write(perf_ctl_fd, "enable\n", 8);
    read(perf_ack_fd, ack, 5);
    assert(strcmp(ack, "ack\n") == 0);

    // Compute the fibonacci number
    r = fib(n);

    // Stop the performance counter and read the ack
    write(perf_ctl_fd, "disable\n", 9);
    read(perf_ack_fd, ack, 5);
    assert(strcmp(ack, "ack\n") == 0);

    // Print the result
    printf("Result: %zu\n", r);

    return 0;
}

prog.cpp

#include <array>       // array
#include <cassert>     // assert
#include <cstddef>     // size_t
#include <cstdlib>     // atoi, getenv
#include <cstring>     // strcmp
#include <iostream>    // cout, endl
#include <unistd.h>    // read, write

std::size_t fib(std::size_t n)
{
    if (n == 0)
    {
        return 0;
    }
    else if (n == 1)
    {
        return 1;
    }
    else
    {
        return fib(n - 1) + fib(n - 2);
    }
}

int main(int argc, char* argv[])
{
    // // Make sure we have the right number of arguments
    // if (argc != 3)
    // {
    //     std::cerr << "Received " << (argc - 1) << " arguments, expected 2.\n") << std::endl;
    //     std::cerr << "Usage: " << argv[0] << " <fd1> <fd2>") << std::endl;
    //     return 1;
    // }
    // perf_ctl_fd = std::atoi(argv[1]);
    // perf_ack_fd = std::atoi(argv[2]);

    int perf_ctl_fd = std::atoi(std::getenv("PERF_CTL_FD"));
    int perf_ack_fd = std::atoi(std::getenv("PERF_ACK_FD"));

    // Sleep for 2 seconds
    sleep(2);

    std::size_t n = 45;
    // std::cin >> n;

    {
        // Start the performance counter and read the ack
        // write enable\n to perf_ctl_fd
        write(perf_ctl_fd, "enable\n", 8);
        std::array<char, 5> ack;
        read(perf_ack_fd, ack.data(), 5);
        assert(std::strcmp(ack.data(), "ack\n") == 0);
    }

    // Compute the fibonacci number
    std::size_t const r = fib(n);

    {
        // Stop the performance counter and read the ack
        // write disable\n to perf_ctl_fd
        write(perf_ctl_fd, "disable\n", 9);

        std::array<char, 5> ack;
        read(perf_ack_fd, ack.data(), 5);
        assert(std::strcmp(ack.data(), "ack\n") == 0);
    }

    // Print the result
    std::cout << "fib(" << n << ") = " << r << std::endl;

    return 0;
}

prog.py

#!/usr/bin/env -S python3 -OO

import os
import sys
import time


def fib(n: int) -> int:
    if n <= 1:
        return n
    return fib(n - 1) + fib(n - 2)


def get_fd_from_env(env_var_name: str) -> int:
    try:
        return int(os.environ[env_var_name])
    except (KeyError, ValueError):
        raise RuntimeError(f"{env_var_name} is not set or is not an integer")


if __name__ == "__main__":
    # # Make sure we have 3 arguments
    # if len(sys.argv) != 3:
    #     print(f"Expected 2 arguments, got {len(sys.argv) - 1}.")
    #     print("Usage: prog.py <n> <output_file>")
    #     sys.exit(1)

    # # Get the arguments
    # perf_ctl_fd = int(sys.argv[1])
    # perf_ack_fd = int(sys.argv[2])

    perf_ctl_fd = get_fd_from_env("PERF_CTL_FD")
    perf_ack_fd = get_fd_from_env("PERF_ACK_FD")

    # Sleep for 2 seconds
    time.sleep(2)

    n = 30

    # Start the performance counter collection
    os.write(perf_ctl_fd, b"enable\n")
    # Wait for the ack
    enabled_ack = os.read(perf_ack_fd, 5).decode("utf-8")
    # print(f"enabled: {enabled_ack!a}")
    if enabled_ack != "ack\n\0":
        print(f"Expected ack\\n, got {enabled_ack!a}")
        sys.exit(1)

    # Run the program
    r = fib(n)

    # Stop the performance counter collection
    os.write(perf_ctl_fd, b"disable\n")
    # Wait for the ack
    disabled_ack = os.read(perf_ack_fd, 5).decode("utf-8")
    # print(f"disabled: {disabled_ack!a}")
    if disabled_ack != "ack\n\0":
        print(f"Expected ack\\n, got {disabled_ack!a}")
        sys.exit(1)

    # Print the result
    print(f"fib({n}) = {r}")

prog.sh

#!/usr/bin/env -S bash -euo pipefail

: ${PERF_CTL_FD:?}
: ${PERF_ACK_FD:?}

fib() {
    local n=$1
    if [[ $n -le 1 ]]; then
        echo $n
    else
        echo $(($(fib $((n - 1))) + $(fib $((n - 2)))))
    fi
}

n=20
# Sleep for 2 seconds
sleep 2

# echo "Sending control message"
echo "enable" >&$PERF_CTL_FD
# echo "Waiting for ACK"
read -r ack <&$PERF_ACK_FD
# echo "Got ACK: $ack"
# Ensure what we read was ack\n
[[ $ack == "ack" ]] || exit 1

r=$(fib $n)

# echo "Sending control message"
echo "disable" >&$PERF_CTL_FD
# echo "Waiting for ACK"
read -r ack <&$PERF_ACK_FD
# echo "Got ACK: $ack"
# Ensure what we read was ack\n
[[ $ack == "ack" ]] || exit 1

echo "fib($n) = $r"

prog2.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define ACK_MSG "ack\n"
#define ACK_LEN strlen(ACK_MSG)

size_t fib(size_t n)
{
    if (n == 0)
        return 0;
    else if (n == 1)
        return 1;
    else
        return fib(n - 1) + fib(n - 2);
}

int main(int argc, char* argv[])
{
    int perf_ctl_fd;
    int perf_ctl_ack_fd;
    char ack[ACK_LEN];
    size_t n;
    size_t r;

    // Get file descriptors from environment variables
    char* ctl_fd_str = getenv("PERF_CTL_FD");
    char* ack_fd_str = getenv("PERF_CTL_ACK_FD");
    if (ctl_fd_str == NULL || ack_fd_str == NULL) {
        fprintf(stderr, "Error: environment variables not set\n");
        return 1;
    }
    perf_ctl_fd = atoi(ctl_fd_str);
    perf_ctl_ack_fd = atoi(ack_fd_str);

    // Get input from user
    char input_buf[256];
    printf("Enter a number: ");
    if (fgets(input_buf, sizeof(input_buf), stdin) == NULL) {
        fprintf(stderr, "Error: could not read input\n");
        return 1;
    }
    n = atoi(input_buf);
    if (n <= 0 || n > 50) {
        fprintf(stderr, "Error: invalid input\n");
        return 1;
    }

    // Start the performance counter and read the ack
    if (write(perf_ctl_fd, "enable\n", 8) != 8) {
        fprintf(stderr, "Error: could not start performance counter\n");
        return 1;
    }
    if (read(perf_ctl_ack_fd, ack, ACK_LEN) != ACK_LEN
        || strcmp(ack, ACK_MSG) != 0) {
        fprintf(stderr, "Error: unexpected acknowledgement\n");
        return 1;
    }

    // Compute the fibonacci number
    r = fib(n);

    // Stop the performance counter and read the ack
    if (write(perf_ctl_fd, "disable\n", 9) != 9) {
        fprintf(stderr, "Error: could not stop performance counter\n");
        return 1;
    }
    if (read(perf_ctl_ack_fd, ack, ACK_LEN) != ACK_LEN
        || strcmp(ack, ACK_MSG) != 0) {
        fprintf(stderr, "Error: unexpected acknowledgement\n");
        return 1;
    }

    // Print the result
    printf("Result: %zu\n", r);

    return 0;
}

prog_fancy.cpp

#include <array>       // for array
#include <cstddef>     // for size_t
#include <cstring>     // for strcmp
#include <iostream>    // for cout, cerr, endl
#include <unistd.h>    // for sleep, read, write

std::size_t fib(std::size_t n)
{
    if (n == 0)
    {
        return 0;
    }
    else if (n == 1)
    {
        return 1;
    }
    else
    {
        return fib(n - 1) + fib(n - 2);
    }
}

struct perf_ipc_context_t
{
    int perf_ctl_fd;
    int perf_ack_fd;

    perf_ipc_context_t() = delete;

    perf_ipc_context_t(int perf_ctl_fd, int perf_ack_fd)
      : perf_ctl_fd(perf_ctl_fd)
      , perf_ack_fd(perf_ack_fd)
    {
    }

    perf_ipc_context_t(perf_ipc_context_t const&) = delete;
    perf_ipc_context_t& operator=(perf_ipc_context_t const&) = delete;

    perf_ipc_context_t(perf_ipc_context_t&& other)
      : perf_ctl_fd(other.perf_ctl_fd)
      , perf_ack_fd(other.perf_ack_fd)
    {
        other.perf_ctl_fd = -1;
        other.perf_ack_fd = -1;
    }
    perf_ipc_context_t& operator=(perf_ipc_context_t&& other)
    {
        perf_ctl_fd = other.perf_ctl_fd;
        perf_ack_fd = other.perf_ack_fd;
        other.perf_ctl_fd = -1;
        other.perf_ack_fd = -1;
        return *this;
    }

    int receive_ack()
    {
        std::array<char, 5> ack_msg;

        // Raead ack from perf_ack_fd
        auto const nread = read(perf_ack_fd, ack_msg.data(), ack_msg.size());
        // Ensure that we read something
        if (nread == -1)
        {
            std::cerr << "Error reading from perf_ack_fd" << std::endl;
            return 1;
        }
        // Ensure that we read the correct amount of bytes
        if (nread != ack_msg.size())
        {
            std::cerr << "Error: read " << nread << " bytes instead of "
                      << ack_msg.size() << std::endl;
            return 1;
        }
        // Ensure what we read was ack\n
        auto const expected_msg = "ack\n";
        if (std::strcmp(ack_msg.data(), expected_msg) != 0)
        {
            std::cerr << "Error: read '" << ack_msg.data() << "' instead of '"
                      << expected_msg << "'" << std::endl;
            return 1;
        }

        return 0;
    }

    void send_cmd(char const msg[])
    {
        // Write msg to perf_ctl_fd
        auto const nwritten = write(perf_ctl_fd, msg, std::strlen(msg) + 1);
        // Ensure that we wrote something
        if (nwritten == -1)
        {
            std::cerr << "Error writing to perf_ctl_fd" << std::endl;
            std::exit(1);
        }
        // Ensure that we wrote the correct amount of bytes
        if (nwritten != std::strlen(msg) + 1)
        {
            std::cerr << "Error: wrote " << nwritten << " bytes instead of "
                      << std::strlen(msg) + 1 << std::endl;
            std::exit(1);
        }
    }

    // Send "enable" to perf via perf_ctl_fd, then read "ack" from perf via perf_ack_fd
    int enable()
    {
        // Send "enable" to perf via perf_ctl_fd
        send_cmd("enable\n");
        // Read "ack" from perf via perf_ack_fd
        return receive_ack();
    }

    // Send "disable" to perf via perf_ctl_fd, then read "ack" from perf via perf_ack_fd
    int disable()
    {
        // Send "disable" to perf via perf_ctl_fd
        send_cmd("disable\n");
        // Read "ack" from perf via perf_ack_fd
        return receive_ack();
    }
};

struct perf_ipc_scope_t
{
    perf_ipc_context_t& ctx;

    perf_ipc_scope_t(perf_ipc_context_t& ctx)
      : ctx(ctx)
    {
        if (ctx.enable() != 0)
        {
            std::cerr << "Error: failed to enable perf" << std::endl;
            std::exit(1);
        }
    }

    ~perf_ipc_scope_t()
    {
        if (ctx.disable() != 0)
        {
            std::cerr << "Error: failed to disable perf" << std::endl;
            std::exit(1);
        }
    }
};

int get_fd_from_env(char const env_var_name[])
{
    auto const env_var = std::getenv(env_var_name);
    if (env_var == nullptr)
    {
        std::cerr << "Error: " << env_var_name << " not set" << std::endl;
        std::exit(1);
    }

    auto const fd = std::atoi(env_var);
    if (fd < 0)
    {
        std::cerr << "Error: " << env_var_name << " is negative" << std::endl;
        std::exit(1);
    }

    return fd;
}

int main(int argc, char* argv[])
{
    std::size_t n = 0;
    n = 45;
    // std::cin >> n;

    // if (argc < 3)
    // {
    //     std::cout << "Usage: prog <perf_ctl_fd> <perf_ack_fd>\n";
    //     return 1;
    // }

    // Get PERF_CTL_FD and PERF_ACK_FD from the environment, check that they are
    // valid, and convert them to integers
    int const perf_ctl_fd = get_fd_from_env("PERF_CTL_FD");
    int const perf_ack_fd = get_fd_from_env("PERF_ACK_FD");

    perf_ipc_context_t perf_ctx{perf_ctl_fd, perf_ack_fd};

    // Sleep for 2 seconds
    sleep(2);

    std::size_t r = 0;
    {
        perf_ipc_scope_t perf_scope(perf_ctx);
        r = fib(n);
    }

    // Display result
    std::cout << "fib(" << n << ") = " << r << std::endl;

    return 0;
}

run.sh

#!/usr/bin/env bash

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd $ROOT_DIR

echo ============================================================
echo Building binaries
echo ------------------------------------------------------------
# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
# cmake --build build

mkdir -p build
# gcc -O3 -g -march=native prog.c -o build/prog
# g++ -O3 -g -march=native prog.cpp -o build/prog
g++ -O3 -g -march=native prog_fancy.cpp -o build/prog
# ln -sf $ROOT_DIR/prog.py build/prog
# ln -sf $ROOT_DIR/prog.sh build/prog
echo ============================================================

echo
echo ============================================================
echo Creating named pipes
echo ------------------------------------------------------------
# Reference: https://man7.org/linux/man-pages/man1/perf-stat.1.html#:~:text=3%20%2D%2Dappend%20%E2%80%94%20%24cmd-,%2D%2Dcontrol%3Dfifo%3Actl,-%2Dfifo%5B%2Cack%2Dfifo
[[ -p ctl_fd.fifo ]] && unlink ctl_fd.fifo
mkfifo ctl_fd.fifo
exec {ctl_fd}<>ctl_fd.fifo
echo ctl_fd: $ctl_fd

# NOTE: ACK is optional (--control fd:${ctl_fd},${ctl_fd_ack} to perf-stat)
[[ -p ctl_fd_ack.fifo ]] && unlink ctl_fd_ack.fifo
mkfifo ctl_fd_ack.fifo
exec {ctl_fd_ack}<>ctl_fd_ack.fifo
# echo ctrl_fd_ack: $ctl_fd_ack
echo ============================================================

echo
echo ============================================================
echo Running perf stat with control pipes
echo ------------------------------------------------------------
# Run perf stat with control pipes
PERF_CTL_FD=$ctl_fd PERF_ACK_FD=$ctl_fd_ack perf stat --delay=-1 --control fd:${ctl_fd},${ctl_fd_ack} -o perf-stat-with-fd.log -- build/prog

echo ============================================================

echo
echo ============================================================
echo Running perf record with control pipes
echo ------------------------------------------------------------
# Run perf record with control pipes
PERF_CTL_FD=$ctl_fd PERF_ACK_FD=$ctl_fd_ack perf record --delay=-1 --control fd:${ctl_fd},${ctl_fd_ack} -o perf-record-with-fd.data -- build/prog
# Display the perf report
perf report -i perf-record-with-fd.data >perf-record-with-fd.log
echo ============================================================

echo
echo ============================================================
echo Closing and removing the named pipes
echo ------------------------------------------------------------
# Close the control pipe
exec {ctl_fd}>&-
exec {ctl_fd_ack}>&-

# Remove the named pipes
[[ -p ctl_fd.fifo ]] && unlink ctl_fd.fifo
[[ -p ctl_fd_ack.fifo ]] && unlink ctl_fd_ack.fifo
echo ============================================================