cri-bench-raw-data.log

System info:
Linux 168.119.165.86 6.1.0-21-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.90-1 (2024-05-03) x86_64 GNU/Linux
Distributor ID:	Debian
Description:	Debian GNU/Linux 12 (bookworm)
Release:	12
Codename:	bookworm
Will run crun runc runsc youki for 1000 times
-----------------------------------
Running: nerdctl run  --network=none --rm --runtime crun busybox true
crun version 1.15
commit: e6eacaf4034e84185fd8780ac9262bbf57082278
rundir: /run/user/0/crun
spec: 1.0.0
+SYSTEMD +SELINUX +APPARMOR +CAP +SECCOMP +EBPF +CRIU +YAJL
The benchmark for crun took 306 seconds
Err: 0
Mean: .28
Min: 0.13
Max: 0.41
-----------------------------------
Running: nerdctl run  --network=none --rm --runtime runc busybox true
runc version 1.1.13
commit: v1.1.13-0-g58aa9203-dirty
spec: 1.0.2-dev
go: go1.21.11
libseccomp: 2.5.5
The benchmark for runc took 362 seconds
Err: 1
Mean: .34
Min: 0.16
Max: 0.44
-----------------------------------
Running: nerdctl run  --network=none --rm --runtime runsc busybox true
runsc version release-20240617.0
spec: 1.1.0-rc.1
The benchmark for runsc took 642 seconds
Err: 0
Mean: .62
Min: 0.44
Max: 1.64
-----------------------------------
Running: nerdctl run  --network=none --rm --runtime youki busybox true
youki version 0.3.3
commit: 0.3.3-0-c03edea
The benchmark for youki took 354 seconds
Err: 36
Mean: .32
Min: 0.16
Max: 0.46
-----------------------------------
All done!

nerdctl run --network=none --memory 160kb --rm --runtime crun busybox true # min 160kb
nerdctl run --network=none --memory 8mb --rm --runtime runc busybox true # min 8mb
nerdctl run --network=none --memory 32mb --rm --runtime runsc busybox true # min 32mb
nerdctl run --network=none --memory 2mb --rm --runtime youki busybox true # min 2mb

cri-bench-raw-errors.log

Errors:
time="2024-06-22T13:16:55Z" level=error msg="forward signal child exited" error="ttrpc: closed: unknown"
0.36
time="2024-06-22T13:32:53Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.22
time="2024-06-22T13:32:54Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.22
time="2024-06-22T13:32:56Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.14
time="2024-06-22T13:33:04Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.18
time="2024-06-22T13:33:12Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.19
time="2024-06-22T13:33:23Z" level=error msg="failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container 88883548369aa91be047a519708e2b3952803bf0525b35b41f4520722e2b7fd1\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container 88883548369aa91be047a519708e2b3952803bf0525b35b41f4520722e2b7fd1\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
0.30
time="2024-06-22T13:33:50Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.22
time="2024-06-22T13:33:50Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.20
time="2024-06-22T13:34:04Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.17
time="2024-06-22T13:34:15Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.18
time="2024-06-22T13:34:15Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.17
time="2024-06-22T13:34:15Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.18
time="2024-06-22T13:34:21Z" level=error msg="failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container b9392b4a68fb11fe434723a084d52168e52f3ff52735736e7e426dadcb516ffb\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container b9392b4a68fb11fe434723a084d52168e52f3ff52735736e7e426dadcb516ffb\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
0.41
time="2024-06-22T13:34:29Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.23
time="2024-06-22T13:34:29Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.20
time="2024-06-22T13:34:29Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.22
time="2024-06-22T13:34:49Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.22
time="2024-06-22T13:34:55Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.18
time="2024-06-22T13:35:40Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.20
time="2024-06-22T13:35:40Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.19
time="2024-06-22T13:36:01Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.21
time="2024-06-22T13:36:29Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.18
time="2024-06-22T13:36:44Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.18
time="2024-06-22T13:36:47Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.19
time="2024-06-22T13:36:48Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.17
time="2024-06-22T13:36:48Z" level=error msg="failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container 1eea527af4dda0e8872097ab9cd461a2947c2f715daae4ae26d3854e128e5210\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container 1eea527af4dda0e8872097ab9cd461a2947c2f715daae4ae26d3854e128e5210\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
time="2024-06-22T13:36:48Z" level=warning msg="failed to remove container 1eea527af4dda0e8872097ab9cd461a2947c2f715daae4ae26d3854e128e5210" error="failed to delete task 1eea527af4dda0e8872097ab9cd461a2947c2f715daae4ae26d3854e128e5210: failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container 1eea527af4dda0e8872097ab9cd461a2947c2f715daae4ae26d3854e128e5210\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container 1eea527af4dda0e8872097ab9cd461a2947c2f715daae4ae26d3854e128e5210\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
0.21
time="2024-06-22T13:37:15Z" level=error msg="failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container 9a47f4db668b026ed090089b4b6c593525550e5f8bc1847690e788edc771d1eb\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container 9a47f4db668b026ed090089b4b6c593525550e5f8bc1847690e788edc771d1eb\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
0.33
time="2024-06-22T13:37:23Z" level=error msg="failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container dd0327f6f5f743b3a6e3481361aa595227b0419b0ddabc104bf0d6b63a273118\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container dd0327f6f5f743b3a6e3481361aa595227b0419b0ddabc104bf0d6b63a273118\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
time="2024-06-22T13:37:23Z" level=warning msg="failed to remove container dd0327f6f5f743b3a6e3481361aa595227b0419b0ddabc104bf0d6b63a273118" error="failed to delete task dd0327f6f5f743b3a6e3481361aa595227b0419b0ddabc104bf0d6b63a273118: failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container dd0327f6f5f743b3a6e3481361aa595227b0419b0ddabc104bf0d6b63a273118\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container dd0327f6f5f743b3a6e3481361aa595227b0419b0ddabc104bf0d6b63a273118\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
0.32
time="2024-06-22T13:37:25Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.21
time="2024-06-22T13:37:34Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.24
time="2024-06-22T13:37:40Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.21
time="2024-06-22T13:37:44Z" level=error msg="failed to delete task: failed to delete task: youki did not terminate successfully: exit status 1: error in executing command: failed to delete container fc11fcce7b6ade9d89f62ac28d4e22da44f38aeb35400ddc9af506b706384d59\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\nError: failed to delete container fc11fcce7b6ade9d89f62ac28d4e22da44f38aeb35400ddc9af506b706384d59\n\nCaused by:\n    0: systemd error: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    1: dbus error: dbus function call error: expected to get a reply for method call, didn't get any\n    2: dbus function call error: expected to get a reply for method call, didn't get any\n: unknown"
0.32
time="2024-06-22T13:37:44Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.15
time="2024-06-22T13:37:49Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.20
time="2024-06-22T13:38:12Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.20
time="2024-06-22T13:38:37Z" level=fatal msg="failed to create shim task: OCI runtime create failed: youki did not terminate successfully: exit status 1: unknown"
Command exited with non-zero status 1
0.21

cri-bench.sh

#!/bin/bash
# set -e -o pipefail
# NOTE: The up to date version is in the playbooks dir repo: https://github.com/hegerdes/ansible-playbooks

REPETITIONS=1000
SLEEP_TIME=120s
RUNTIMES=("crun" "runc" "runsc" "youki")
# CRI_EXTRA_ARGS="--memory 4mb"

# Make sure tools are installed
apt-get install -qq -y bc jq curl time lsb-release >/dev/null

# Tool setup
if ! command -v nerdctl >/dev/null 2>&1; then
    echo "Installing nerdctl!"
    if [ "$(uname -m)" = "x86_64" ]; then
        ARCH=amd64
    elif [ "$(uname -m)" = "aarch64" ]; then
        ARCH=arm64
    fi
    NERDCTL_DEFAULT_VERSION=$(curl -sL https://api.github.com/repos/containerd/nerdctl/releases/latest | jq -r .name)
    NERDCTL_VERSION="${NERDCTL_VERSION-$NERDCTL_DEFAULT_VERSION}"
    if echo "${NERDCTL_VERSION}" | grep -q "v"; then
        NERDCTL_VERSION="${NERDCTL_VERSION:1}"
    fi
    curl -sL --output /tmp/nerdctl.tar.gz "https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${ARCH-amd64}.tar.gz"
    tar -xzf /tmp/nerdctl.tar.gz -C /usr/local/bin/ --exclude={containerd-rootless-setuptool.sh,containerd-rootless.sh}
    nerdctl --version
fi
nerdctl pull --quiet busybox >/dev/null
# ctr i pull docker.io/library/busybox:latest >/dev/null
# ctr run $CRI_EXTRA_ARGS --rm --runc-binary crun --runtime io.containerd.runc.v2 docker.io/library/busybox:latest true

echo -e "System info:\n$(uname -a)\n$(lsb_release -a)"
echo "Errors:" >error.log
echo "Will run ${RUNTIMES[*]} for $REPETITIONS times"
echo "-----------------------------------"
for rt in "${RUNTIMES[@]}"; do
    cmd="nerdctl run $CRI_EXTRA_ARGS --network=none --rm --runtime $rt busybox true"
    echo "Running: $cmd"
    bash -c "${rt} --version"
    total=0
    min=999999999
    max=0
    errors=0
    SECONDS=0
    bash -c "${cmd}" 2 >/dev/null >&1
    for ((i = 1; i <= REPETITIONS; i++)); do
        result=$(/usr/bin/time -f "%e" -o /dev/stdout -a $cmd 2>&1)
        if [ "${#result}" -gt 5 ]; then
            echo "${result}" >>error.log
            errors=$(echo $errors+1 | bc)
        else
            total=$(echo $total+$result | bc)
            if (($(echo "$result < $min" | bc -l))); then
                min=$result
            fi
            if (($(echo "$result > $max" | bc -l))); then
                max=$result
            fi
        fi
        # sleep 1s
    done
    echo "The benchmark for ${rt} took $SECONDS seconds"
    mean=$(echo "scale=2; $total / $REPETITIONS" | bc)

    echo "Err: $errors"
    echo "Mean: $mean"
    echo "Min: $min"
    echo "Max: $max"
    echo "-----------------------------------"
    sleep $SLEEP_TIME
done
echo "All done!"