Install NVIDIA drivers and container toolkit on Debian

install-nvidia-drivers-debian.sh

# check running kernel version
uname -r

# check installed kernel headers
apt list --installed | grep "linux-headers"

# if the correct ones aren't installed, then install them
sudo apt install linux-headers-$(uname -r)

# probably reboot, but I'm not sure if it's necessary

# just to make sure a clean start, remove any existing nvidia packages
sudo apt purge nvidia.

# check hardware to make sure the GPUs show up at all
sudo lspci | grep VGA

# check kernel driver. should be something like: /dev/dri/cardXYZ
find /dev -group video

# Show the current GPU driver. It is in the "configuration" section.
# If "driver=nvidia" is not there, then the nvidia driver is not used. It might be "nouveau" or not present at all.
sudo apt install lshw
lshw -C display

# detect what version of the nvidia drivers to use
sudo apt install nvidia-detect
nvidia-detect

# install whichever drivers it recommended
sudo apt install nvidia-legacy-390xx-driver 

# I'm not sure if this is necessary, but specify to use nvidia drivers
sudo /usr/sbin/update-glx --config nvidia

# probably reboot again, but I'm not sure if it is needed here, either

# check that  the nvidia driver is used
lshw -C display

# install the nvidia system management interface, and check that it sees the GPUs
# this package needs to match the installed drivers
sudo apt install nvidia-legacy-390xx-smi
nvidia-smi

# If there is an error, then something went wrong.

# then follow the container tool set up: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html

distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
   && curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - \
   && curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
   
sudo apt-get install -y nvidia-docker2

sudo systemctl restart docker

# most recent version that the 390xx legacy drivers support
sudo docker run --rm --gpus all nvidia/cuda:9.1-cudnn7-runtime nvidia-smi