In case your latest Ubuntu update breaks everything for CUDA. :p

install_cuda.sh

# From https://heads0rtai1s.github.io/2021/02/25/gpu-setup-r-python-ubuntu/


# Fresh install
sudo apt-get --purge remove "*cublas*" "*cufft*" "*curand*" "*cusolver*" "*cusparse*" "*npp*" "*nvjpeg*" "cuda*" "nsight*"
sudo apt-get --purge remove "*nvidia*"
sudo apt-get autoremove

# Drivers
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget https://developer.download.nvidia.com/compute/cuda/11.0.3/local_installers/cuda-repo-ubuntu2004-11-0-local_11.0.3-450.51.06-1_amd64.deb
sudo dpkg -i cuda-repo-ubuntu2004-11-0-local_11.0.3-450.51.06-1_amd64.deb
sudo apt-key add /var/cuda-repo-ubuntu2004-11-0-local/7fa2af80.pub
sudo apt-get update
sudo apt-get install cuda



# Optional
sudo apt-get install g++ freeglut3-dev build-essential libx11-dev libxmu-dev libxi-dev libglu1-mesa libglu1-mesa-dev
sudo dpkg -i libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb 
sudo dpkg -i libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb 
sudo dpkg -i libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb 

It seems sudo ubuntu-drivers install was enough in my case.