Setup for XRDP using Nvidia Acceleration

xrdp-nvidia-setup.sh

#!/usr/bin/env bash

set -e

cd ~

sudo -v

# Make sure system is in a good, updated, clean, state.
sudo apt-get -y update
sudo apt-get -y dist-upgrade
sudo apt-get -y autoremove

# Install CUDA (Shamelessly copied from Nvidia's website):
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update
sudo apt-get -y install cuda

sudo usermod $USER -a -G video
sudo usermod $USER -a -G tty
sudo usermod $USER -a -G render

# XRDP Build Pre-reqs Part 1
sudo apt-get -y install git autoconf libtool pkg-config gcc g++ make \
    libssl-dev libpam0g-dev libjpeg-dev libx11-dev libxfixes-dev libxrandr-dev \
    flex bison libxml2-dev intltool xsltproc xutils-dev python3-libxml2 g++ \
    xutils libfuse-dev libmp3lame-dev nasm libpixman-1-dev xserver-xorg-dev \
    libjson-c-dev libsndfile1-dev libspeex-dev libspeexdsp-dev libpulse-dev \
    libpulse0 autopoint \*turbojpeg\* libfdk-aac-dev libopus-dev libgbm-dev \
    libx264\* libx264-dev
 
#XRDP Build Pre-reqs Part 2 (For some reason apt needs this to be separate)
sudo apt-get -y install libepoxy-dev
 
#System Pre-reqs
sudo apt-get -y install xorg xserver-xorg-legacy dbus-x11
sudo apt-get -y remove dbus-user-session

sudo tee /etc/X11/Xwrapper.config > /dev/null << EOL
# Xwrapper.config (Debian X Window System server wrapper configuration file)
#
# This file was generated by the post-installation script of the
# xserver-xorg-legacy package using values from the debconf database.
#
# See the Xwrapper.config(5) manual page for more information.
#
# This file is automatically updated on upgrades of the xserver-xorg-legacy
# package *only* if it has not been modified since the last upgrade of that
# package.
#
# If you have edited this file but would like it to be automatically updated
# again, run the following command as root:
#   dpkg-reconfigure xserver-xorg-legacy 
needs_root_rights=no
allowed_users=anybody
EOL

echo "gnome-session --session=ubuntu" > ~/.xsession
sudo chmod +x ~/.xsession
sudo tee ~/.xsessionrc > /dev/null << EOL
export DESKTOP_SESSION=ubuntu
export GNOME_SHELL_SESSION_MODE=ubuntu
export XDG_CURRENT_DESKTOP=ubuntu:GNOME
EOL
sudo chmod +x ~/.xsessionrc

BUILD_DIR=$(mktemp -d)

echo "Building xrdp..."
git clone https://github.com/Nexarian/xrdp.git --branch mainline_merge "$BUILD_DIR/xrdp"
cd "$BUILD_DIR/xrdp"
./bootstrap
./configure --enable-fuse --enable-rfxcodec --enable-pixman --enable-mp3lame \
    --enable-sound --enable-opus --enable-fdkaac --enable-x264 --enable-nvenc
make -j $(nproc) clean all
sudo make install

echo "Building xorgxrdp..."
git clone https://github.com/Nexarian/xorgxrdp.git --branch mainline_merge "$BUILD_DIR/xorgxrdp"
cd "$BUILD_DIR/xorgxrdp"
./bootstrap
./configure --with-simd --enable-lrandr
make -j $(nproc) clean all
sudo make install

echo "Configuring Nvidia..."
BUS_ID=$(nvidia-smi --query-gpu=pci.bus --format=csv | sed -n '2 p' | xargs -I{} printf "%d\n" {})
# Alternative method to get the BUS ID that doesn't work on some systems (Such as a laptop with an Intel iGPU and an Nvidia Discrete GPU).
# BUS_ID=$(lspci | grep -e 'VGA.*NVIDIA' | cut -d: -f1 | xargs -I{} printf "%d\n" 0x{} | sed -n 1p)
# nvidia-xconfig --query-gpu-info
sudo sed -i -E 's/(BusID "PCI:)[[:digit:]]+(:0:0")/\1'$BUS_ID'\2/' /etc/X11/xrdp/xorg_nvidia.conf

echo "Starting the server..."
sudo systemctl enable xrdp
sudo service xrdp start

@alexanderzjs Thanks for your post. I got it working on my local test machine with xserver-xorg-video-dummy and by adding a monitor to xorg_nvidia.conf. It still does not work on the server, but I think it is a GPU/cuda/driver issue.

Great! Adding a virtual monitor is definitely the solution.

Hi @Nexarian, thank you for the Nvidia GPU setup. It works for me. However, I get 2 Xorg and gnome-shell processes at the same time when checking nvidia-smi. Is that normal? Is it any possible to optimize this one? Many thanks.