Memory profiling for dbt-core versions

profile-dbt-versions.sh

#!/bin/bash
set -euxo pipefail

###############################################################################
# Notes:
# - Caveat emptor. Script is presented as is, no warranty expressed, etc.
# - Requires uv to be installed
# - Set the environment variables + any other env vars you need to run dbt.
# - This script will delete your .venv; be careful. Set env vars correctly!
###############################################################################

export VIRTUAL_ENV=.venv
export DBT_PROJECT_DIR=.
export DBT_PROFILES_DIR=./profiles

# Try one of: "parse" "compile", or "--help"
export DBT_CMD=--help

# Put other env vars here:
export SNOWFLAKE_USER=x
export SNOWFLAKE_PASSWORD=x
export SNOWFLAKE_ROLE=x
export SNOWFLAKE_DATABASE=x
export SNOWFLAKE_WAREHOUSE=x

###############################################################################

uv venv "${VIRTUAL_ENV}"

versions=("1.8.0b1" "1.7" "1.6")

for ver in "${versions[@]}"
do
  echo "Running ver=${ver}"

  rm -rf "${VIRTUAL_ENV}/lib/python3.11/site-packages" || true
  mkdir -p "${VIRTUAL_ENV}/lib/python3.11/site-packages"
  rm "${DBT_PROJECT_DIR}/target/partial_parse.msgpack" || true

  uv pip install "dbt-core==${ver}" "dbt-snowflake" "memory-profiler" "matplotlib"
  "${VIRTUAL_ENV}/bin/mprof" run "${VIRTUAL_ENV}/bin/dbt" "${DBT_CMD}"
  "${VIRTUAL_ENV}/bin/mprof" plot -o "mprof_${ver}.png" -t "mprof for dbt-core==${ver}"
done