Datadog Agent integrations live Python interpreter profiling

README.md

Basic use

Description

• Uses py-spy under the covers. See here for all invocation params.
• Uses shared PID namespaces, SYS_PTRACE capability, and heuristics for PID identification to attach to the appropriate live/unmodified Agent running in a separate container.
• Assumes that the name of the agent container is dd-agent but you can easily modify profiler.sh to match either the container ID or name.

Note: In theory, this tool supports Python up to 3.10 as of this writing (Nov 2021).

UI

"top"

"record" SVG output

With image build

$ # "top"-like dynamic UI
$ ./profiler.sh

$ # With custom options (flamegraph in this case). Output saved in bind-mounted CWD.
$ ./profiler.sh record -d 30 -r 100 -o /output/profile.svg

$ # With custom options (dump callstack in this case)
$ # This is a bit useless for the agent as we run the checks at periodic intervals
$ ./profiler.sh dump

Without image build

$ # "top"-like dynamic UI
$ docker run --rm -it \
             --name="dd-agent-profiler" \
             --pid="container:${AGENT_CONTAINER_ID}" \
             --cap-add="SYS_PTRACE" \
             --security-opt="seccomp=unconfined" \
             --security-opt="apparmor=unconfined" \
             datadog/agent-dev:pyprof

$ # With custom options (flamegraph in this case). Output saved in bind-mounted CWD.
$ docker run --rm -it \
             --name="dd-agent-profiler" \
             --pid="container:${AGENT_CONTAINER_ID}" \
             --cap-add="SYS_PTRACE" \
             --security-opt="seccomp=unconfined" \
             --security-opt="apparmor=unconfined" \
             -v "$(pwd):/output" \
             datadog/agent-dev:pyprof record -d 30 -r 100 -o /output/profile.svg
           
$ # With custom options (dump callstack in this case)
$ # This is a bit useless for the agent as we run the checks at periodic intervals
$ docker run --rm -it \
             --name="dd-agent-profiler" \
             --pid="container:${AGENT_CONTAINER_ID}" \
             --cap-add="SYS_PTRACE" \
             --security-opt="seccomp=unconfined" \
             --security-opt="apparmor=unconfined" \
             datadog/agent-dev:pyprof dump

Dockerfile

FROM python:3.8-slim-bullseye

RUN apt update && \
    DEBIAN_FRONTEND="noninteractive" apt install -y procps && \
    apt clean

RUN pip3 install py-spy==0.3.11

RUN mkdir -p /output

ENTRYPOINT [ "/python_pprof.sh" ]
CMD []

COPY python_pprof.sh /

profiler.sh

#!/bin/bash -e

set -euo pipefail

IMAGE_NAME=datadog/agent-dev:pyprof
AGENT_CONTAINER_ID="dd-agent"

squash="no"
if docker version | grep -e "Experimental:" -e "true$" >/dev/null; then
  echo "Enabling squashed image..."
  squash="yes"
fi

if [[ "$squash" == "yes" ]]; then
  docker build --squash -t "$IMAGE_NAME" .
else
  docker build -t "$IMAGE_NAME" .
fi


# XXX: SYS_ADMIN may be required for some other debug tools
#           --cap-add="SYS_ADMIN" \
docker run --rm -it \
           --name="dd-agent-profiler" \
           --pid="container:${AGENT_CONTAINER_ID}" \
           --cap-add="SYS_PTRACE" \
           --security-opt="seccomp=unconfined" \
           --security-opt="apparmor=unconfined" \
           -v "$(pwd):/output" \
           "$IMAGE_NAME" "$@"

python_pprof.sh

#!/bin/bash

set -euo pipefail

agent_pid=$(pgrep -x agent)

if [ $# -eq 0 ]; then
  echo "Agent PID: $agent_pid"

  echo "Running profiler..."
  py-spy top --pid "$agent_pid"
fi

if [ "$1" == "bash" ]; then
  bash
  exit 0
fi

echo "Running custom profiler ($*)..."
py-spy "$@" --pid "$agent_pid"