slow bob with many root recipes

gistfile1.txt

After removing the *rootness* of all root recipes beside a very simple
one, add it back one recipe file at a time and profile the build of
the simple recipe.

New version also adds a second run with hot cache.

profile-roots.sh

#!/bin/bash

command=${1:-profile}

BOB=$(type -p bob)
SIMPLERECIPE="build_tests::bob-project-dir"
case $command in
    profile|deactivate)
        SIMPLEFNAME=$(bob query-recipe "$SIMPLERECIPE" 2>/dev/null | head -n 1)
        ;;
esac


numOfRootrecipes() {
    local INCLUDE=("--include=*.yaml")
    if [[ $1 = "--all" ]]; then
        INCLUDE+=("--include=*.yaml.root")
    fi
    grep -r  "${INCLUDE[@]}" -E -e "root:\s+true" recipes/* | wc -l
}


doProfile() {
    local num
    num="$(printf "%03d" "$(numOfRootrecipes)")"
    rm -rf .bob-{packages,tree}* dev
    local time
    local time2
    time=$( (time -p python3 -m cProfile -o "bob-recipes-$num.stats" "$BOB" \
                  dev -f "$SIMPLERECIPE" >/dev/null 2>&1) 2>&1 \
                | grep user | cut -f 2 -d " ")
    # now a second run with hot cache
    time2=$( (time -p python3 -m cProfile -o "bob-recipes-$num.stats" "$BOB" \
                   dev -f "$SIMPLERECIPE" >/dev/null 2>&1) 2>&1 \
                 | grep user | cut -f 2 -d " ")
    printf "%s\t%s\n" "$time" "$time2"
}


rootRecipesOff() {
    while IFS= read -r fname
    do
        #echo "$fname"
        if [[ $fname == "$SIMPLEFNAME" ]]; then continue; fi
        sed -i.root -e 's/root:\s\+true/root: false/' "$fname"
    done < <(grep -rl --include="*.yaml" -E -e "root:\s+true" recipes/*)
}


# undo non-rootness from all recipes found
rootRecipesOn() {
    local recipe
    while IFS= read -r recipe
    do
        local n="${recipe%.root}"
        mv -f "$recipe" "$n"
        echo "$n"
    done < <(find recipes -name "*.yaml.root")
}


# undo non-rootness from first recipe found
rootRecipeOn() {
    local recipe
    recipe=$(find recipes -name "*.yaml.root" | head -n 1)
    if [[ -z "$recipe" ]]; then
        return 1
    else
        local n="${recipe%.root}"
        mv -f "$recipe" "$n"
        echo "$n"
    fi
}


case $command in
    profile)
        numAll=$(numOfRootrecipes --all)
        currRecipe=$SIMPLEFNAME
        rootRecipesOff
        while true; do
            num=$(numOfRootrecipes)
            printf "%s\t%s\t%s\t" "$(printf "%03d" "$num")" "$numAll" "$currRecipe"
            doProfile
            currRecipe=$(rootRecipeOn) || break
        done
        ;;
    single)
        numAll=$(numOfRootrecipes --all)
        num=$(numOfRootrecipes)
        printf "%s\t%s\t" "$(printf "%03d" "$num")" "$numAll"
        doProfile
        ;;
    deactivate)
        rootRecipesOff
        ;;
    count)
        numOfRootrecipes
        ;;
    activate-all)
        rootRecipesOn
        ;;
    *)
        1>&2 echo "unknown command '$command'"
        ;;
esac