experimental Installation script for bitsandbytes-rocm for oobabooga/text-generation-ui for ubuntu 22.04

bitsandbytes-rocm-installer.sh

#!/bin/bash

echo "This script is completely untested. "
echo "It's more intended as a general overview of what needs to be done to install bitsandbytes-rocm in the text-generation-ui env on Ubuntu 22.04"
echo "Run from the same directory as start-webui.sh"

# make sure user wants to continue
read -p "Do you want to continue? [y/N] " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
    exit 1
fi

echo "Assuming ROCm is already installed. Attempting to configure it for compilation..."
sudo amdgpu-install --usecase=hiplibsdk,rocm

echo "Installing compiler dependencies..."
sudo apt install libstdc++-12-dev build-essential

#source conda env
INSTALL_ENV_DIR="$(pwd)/installer_files/env"
export PATH="$INSTALL_ENV_DIR/bin:$PATH"
CONDA_BASEPATH=$(conda info --base)
source "$CONDA_BASEPATH/etc/profile.d/conda.sh" # otherwise conda complains about 'shell not initialized' (needed when running in a script)
conda activate

echo "Uninstalling old version of bitsandbytes if installed..."
pip3 uninstall bitsandbytes
pip3 uninstall bitsandbytes-rocm

echo "Downloading bitsandbytes-rocm source..."
git clone https://git.ecker.tech/mrq/bitsandbytes-rocm
cd bitsandbytes-rocm || exit

echo "Compiling bitsandbytes-rocm..."
# set CUDA_VERSION to gfx1030 if not set. This is the default for 6XXX series cards
CUDA_VERSION=${CUDA_VERSION:-gfx1030}
export CUDA_VERSION
make hip
python setup.py install
cd ..
rm -rf bitsandbytes-rocm

echo "Done. You should hopefully be able to run server.py with the --load-in-8bit flag now."