/etc/modprobe.d/nvidia.conf

nvidia.conf

# https://wiki.archlinux.org/title/NVIDIA/Tips_and_tricks
# https://forums.developer.nvidia.com/t/does-modprobe-d-nvidia-conf-nvreg-enablestreammemops-need-to-be-set-for-gpudirect-to-work/79511
# https://github.com/NVIDIA/open-gpu-kernel-modules/blob/main/src/nvidia/interface/nvrm_registry.h
# Requires nvidia.ko >= 510.40.3 loaded with PeerMappingOverride=1. https://docs.nvidia.com/nvshmem/install-guide/index.html
# https://wiki.gentoo.org/wiki/NVIDIA/nvidia-drivers/en
# https://github.com/ventureoo/nvidia-tweaks
# https://nvdam.widen.net/s/k8vrp9xkft/tech-overview-magnum-io-1790750-r5-web
# https://developer.download.nvidia.com/devzone/devcenter/cuda/docs/GPUDirect_Technology_Overview.pdf
# https://docs.nvidia.com/nvshmem/pdf/NVSHMEM-Installation-Guide.pdf

# /usr/src/nvidia-530.41.03/nvidia/nv-reg.h
# __NV_USE_PAGE_ATTRIBUTE_TABLE
# NVreg_UsePageAttributeTable=1 (Default 0) - Activating the better
# memory management method (PAT). The PAT method creates a partition type table
# at a specific address mapped inside the register and utilizes the memory architecture
# and instruction set more efficiently and faster.
# If your system can support this feature, it should improve CPU performance.
# 
# __NV_ENABLE_PCIE_GEN3
# NVreg_EnablePCIeGen3=1 (Default 1=new-driver / 0=old-driver) - Enable PCIe Gen 3.x support.
# If your graphics card supports PCIe 3.x, this driver module option
# can active the high-speed 8 GT bus.
# By default the Nvidia driver is set to use PCIe Gen 2.x for compatibility reasons.
# 
# __NV_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS
# NVreg_InitializeSystemMemoryAllocations=0 (Default 1) - Disables
# clearing system memory allocation before using it for the GPU.
# Potentially improves performance, but at the cost of increased security risks.
# Write "options nvidia NVreg_InitializeSystemMemoryAllocations=1" in /etc/modprobe.d/nvidia.conf,
# if you want to return the default value.
# 
# GPUDirect Async
# __NV_ENABLE_STREAM_MEMOPS
# https://docs.nvidia.com/nvshmem/pdf/NVSHMEM-Installation-Guide.pdf
# Requires nvidia.ko >= 510.40.3 loaded with NVreg_RegistryDwords="PeerMappingOverride=1;"
# NVreg_EnableStreamMemOPs=1 (Default 0) - Activates the support for
# CUDA Stream Memory Operations in user-mode applications.
#
# GPUDirect RDMA
# NVreg_EnableResizableBar

#options nvidia NVreg_UsePageAttributeTable=1 NVreg_EnableGpuFirmware=1 NVreg_EnablePCIeGen3=1 NVreg_EnableStreamMemOPs=1 NVreg_RegistryDwords="PeerMappingOverride=1;"
options nvidia NVreg_EnablePCIeGen3=1 NVreg_UsePageAttributeTable=1 NVreg_EnableGpuFirmware=1 NVreg_EnableResizableBar=1