Skip to content

Instantly share code, notes, and snippets.

@peacepenguin
Created December 22, 2022 19:37
Show Gist options
  • Save peacepenguin/6f7b99af75e3c812ebd67177db83ef8c to your computer and use it in GitHub Desktop.
Save peacepenguin/6f7b99af75e3c812ebd67177db83ef8c to your computer and use it in GitHub Desktop.
amd 6800 xt hip rocm and stable diffusion testing on linux
## last tested nov. 2022
## fedora 37 (targetting rhel 9.1 amd repos) (using GUI based Fedora 37, which includes a working AMDGPU kernel driver by default)
## rocm 5.3.3
## note the amdgpu kernel driver in use before starting:
modinfo amdgpu |grep ver
# https://docs.amd.com/bundle/ROCm-Installation-Guide-v5.3.3/page/How_to_Install_ROCm.html#d23e7177
## update the os:
sudo dnf update
## install the ROCM installer package directly from amd's site.
sudo yum install https://repo.radeon.com/amdgpu-install/5.3.3/rhel/9.0/amdgpu-install-5.3.50303-1.el9.noarch.rpm
## point the repo file to RHEL 9.1: (since we're not on RHEL, force the closest RHEL version to fedora, or just use ALMA linux 9.1 to ensure an exact match)
sudo nano /etc/yum.repos.d/amdgpu.repo
# from:
baseurl=https://repo.radeon.com/amdgpu/5.3.3/rhel/$amdgpudistro/main/x86_64
#to:
baseurl=https://repo.radeon.com/amdgpu/5.3.3/rhel/9.1/main/x86_64
## install the rocm bits: (wihtout the DKMS module, fedora already has an up-to-date kernel driver for amdgpu if running the gui)
sudo amdgpu-install --usecase=rocm --no-dkms
## completes successfully on FEDORA 37 - targetting RHEL 9.1 rocm repo!
## reboot
sudo systemctl reboot
## check status of install:
## check for dkms module: (dkms module NOT needed in a gui distro that already has the amdgpu driver module loaded)
dkms status
# check that rocminfo detects the gpu:
/opt/rocm-5.3.3/bin/rocminfo
# check that clinfo detects the gpu:
/opt/rocm-5.3.3/opencl/bin/clinfo
#############################################
## run some basic hip code to validate:
## install git:
sudo dnf install git cmake
sudo mkdir /opt/build
sudo chmod 777 /opt/build
cd /opt/build
git clone https://github.com/ROCm-Developer-Tools/HIP-Examples.git
## run some test code: (see test_all.sh for some of these)
cd /opt/build/HIP-Examples
cd strided-access
make clean
make
./strided-access
cd ..
##
cd reduction
make clean
make
./run.sh
cd ..
##
cd cuda-stream
make clean
make
./stream
cd ..
##
cd vectorAdd
make clean
make
cd ..
##
cd rtm8
./build_hip.sh
./rtm8_hip
cd ..
## compile and run MIXBENCH
cd /opt/build
git clone https://github.com/ekondis/mixbench
cd mixbench
## create a build dir for mixbench:
mkdir build
cd /opt/build/mixbench/build
## add the set_property line to the end of the cmakelists file: (wasn't needed on alma 9.1, but is needed on fedora 37)
nano ../mixbench-hip/CMakeLists.txt
# gfx1030 is radeon 6800 xt (should also be same for 6800 and 6900 xt and 6950 xt)
set_property(TARGET mixbench-hip PROPERTY HIP_ARCHITECTURES gfx1030)
#######################
## for cuda
# install dev tools:
sudo yum groupinstall 'Development Tools'
## install home brew. (not sudo)
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
#==> Next steps:
#- Run these three commands in your terminal to add Homebrew to your PATH:
# echo '# Set PATH, MANPATH, etc., for Homebrew.' >> /home/user11/.bash_profile
# echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"' >> /home/user11/.bash_profile
# eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
#- Install Homebrew's dependencies if you have sudo access:
# sudo yum groupinstall 'Development Tools'
# For more information, see:
# https://docs.brew.sh/Homebrew-on-Linux
#- We recommend that you install GCC:
# brew install gcc
#- Run brew help to get started
##
## add homebrew envs to current shell to get 'brew' available:
## this also makes any apps installed by brew availabe to the current shell.
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
# install needed gcc 11 for cuda:
brew install 'gcc@11'
## link needed bins to the cuda area:
# skip this, we have gcc-11 already in the shell path from 'eval' from brew above.
#sudo su -
#cd /usr/local/bin
#for x in /home/linuxbrew/.linuxbrew/bin/*-11; do ln -s $x .; done
## link the brew installed gcc-11 bits directly into the CUDA folder:
#cd /usr/local/cuda/bin
#ln -s /usr/local/bin/gcc-11 gcc
#ln -s /usr/local/bin/g++-11 g++
sudo ln -s /home/linuxbrew/.linuxbrew/bin/gcc-11 /usr/local/cuda/bin/gcc
sudo ln -s /home/linuxbrew/.linuxbrew/bin/g++-11 /usr/local/cuda/bin/g++
## set the add the cuda/bin folder to the FRONT of the PATH variable for this session (to ensure 'gcc' is v11 from here)
PATH="/usr/local/cuda/bin/:$PATH"
## configure the cmake build for mixbench:
export CUDAARCHS='native'
# or:
#export CUDAARCHS='61' #for titanX
export CUDACXX='/usr/local/cuda/bin/nvcc'
## configure and generate make files:
cmake ../mixbench-cuda
#
#################### END CUDA SPECIFIC ###########
###################################################
## configure and generate make files:
cmake ../mixbench-hip
## compile the HIP mixbench binary:
make
## run the built bits, generates a CSV to std out with the results:
/opt/build/mixbench/build/mixbench-hip
/opt/build/mixbench/build/mixbench-cuda
##########################################################
#### compile and run GPU-STREAM benchmark: (BabelStream)
cd /opt/build
git clone https://github.com/UoB-HPC/BabelStream
## add the ROCM HIP path to the includes_directories variable in CMAKE:
cp /opt/build/BabelStream/CMakeLists.txt ~/CMakeLists.txt.bak
sed -i 's^include_directories(src)^include_directories(src /opt/rocm/hip/include)^' /opt/build/BabelStream/CMakeLists.txt
diff /opt/build/BabelStream/CMakeLists.txt ~/CMakeLists.txt.bak
# configure the build, build type defaults to Release (must manually specifcy the cxx compiler to use for HIP builds) from https://rocmdocs.amd.com/en/latest/Installation_Guide/Using-CMake-with-AMD-ROCm.html
cd /opt/build/BabelStream
cmake -Bbuild -H. -DCMAKE_CXX_COMPILER=/opt/rocm-5.3.3/bin/hipcc -DMODEL=hip
############## or for cuda:#####
cd /opt/build/BabelStream
cmake -Bbuild -H. -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCUDA_ARCH=native -DMODEL=cuda
############ end cuda ######
# compile
cd /opt/build/BabelStream
cmake --build build
# run executables in ./build
# hip:
/opt/build/BabelStream/build/hip-stream
# cuda:
/opt/build/BabelStream/build/cuda-stream
##############################################################
####################### end ROCM examples ####################
########################################################
##### testing stable diffusion:
### linux:
# instructions for stable diffusion on amd
# https://www.videogames.ai/2022/11/06/Stable-Diffusion-AMD-GPU-ROCm-Linux.html
# install conda: (skipping prereqs for now)
# yum install libXcomposite libXcursor libXi libXtst libXrandr alsa-lib mesa-libEGL libXdamage mesa-libGL libXScrnSaver
# manually download the ~1GB installer .sh file:
# https://docs.anaconda.com/anaconda/install/linux/
cd ~/Downloads
chmod +x Anaconda3-2022.10-Linux-x86_64.sh
./Anaconda3-2022.10-Linux-x86_64.sh
# take defaults for the install, it will default to:
# /home/user11/anaconda3
######### end question: no #######
#Do you wish the installer to initialize Anaconda3
#by running conda init? [yes|no]
#[no] >>>
#You have chosen to not have conda modify your shell scripts at all.
#To activate conda's base environment in your current shell session:
#eval "$(/home/user11/anaconda3/bin/conda shell.YOUR_SHELL_NAME hook)"
#To install conda's shell functions for easier access, first activate, then:
#conda init
#If you'd prefer that conda's base environment not be activated on startup,
# set the auto_activate_base parameter to false:
#conda config --set auto_activate_base false
###################
## setup conda into the path temporarily for this session:
source ~/anaconda3/bin/activate
## make sure your in (base) from the source command above first.
cd ~
git clone https://github.com/CompVis/stable-diffusion.git
cd ~/stable-diffusion/
conda env create -f environment.yaml
## create the model directory for sd v1:
cd ~/stable-diffusion
mkdir models/ldm/stable-diffusion-v1/
## link the model:
cd ~/stable-diffusion
## remove a previous sym link if needed:
rm models/ldm/stable-diffusion-v1/model.ckpt
#### pick a model to link as model.ckpt: (aquire the sd-v1-4.ckpt model elsewhere)
## SD 1.4 standard model:
ln -s /home/user11/Downloads/sd-v1-4.ckpt models/ldm/stable-diffusion-v1/model.ckpt
#######
# now activate the new 'ldm' environment created above via 'conda env create...'
conda activate ldm
## install the version of pytorch for rocm: https://pytorch.org/ pick pytorch: stable, linux, pip,rocm to get the proper url/command line to use below:
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.2/
cp ~/stable-diffusion/scripts/txt2img.py ~/stable-diffusion/scripts/txt2img-safe.py
# use it! (first time running downloads about 1.5GB from github)
source ~/anaconda3/bin/activate
conda activate ldm
cd ~/stable-diffusion
# 6 image grid example:
python3 scripts/txt2img.py --prompt "a photograph of an astronaut riding a horse" --plms
# single example:
python3 scripts/txt2img.py --prompt "astronaut riding a horse" --H 512 --W 512 --n_iter 1 --ddim_steps 50 --n_samples 1
# safe example:
python3 scripts/txt2img-safe.py --prompt "astronaut riding a horse" --H 512 --W 512 --n_iter 1 --ddim_steps 50 --n_samples 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment