Kubernetes-ready Nvidia Orin NX flashing for Jetson 36.3 on a turing-pi2

setup.sh

# From shit-box

# Need to modify the host doing the flashing
sudo vi /etc/default/nfs-kernel-server
  RPCMOUNTDOPTS="--manage-gids --no-nfs-version 4"
sudo reboot

sudo apt install -y qemu-user-static make git-core build-essential bc flex bison ncurses-dev libssl-dev gcc-aarch64-linux-gnu

export JETSON_VERSION=36.3

mkdir ~/nvidia
cd ~/nvidia

wget -O bsp.tbz2 -L https://developer.nvidia.com/downloads/embedded/l4t/r36_release_v3.0/release/jetson_linux_r$JETSON_VERSION.0_aarch64.tbz2
wget -O rootfs.tbz2 -L https://developer.nvidia.com/downloads/embedded/l4t/r36_release_v3.0/release/tegra_linux_sample-root-filesystem_r$JETSON_VERSION.0_aarch64.tbz2

tar -xvpf bsp.tbz2
sudo tar -xvpf rootfs.tbz2 -C Linux_for_Tegra/rootfs/

# Turing Pi 2 does not have the onboard EEPROM
sed -i 's/cvb_eeprom_read_size = <0x100>/cvb_eeprom_read_size = <0x0>/g' ~/nvidia/Linux_for_Tegra/bootloader/generic/BCT/tegra234-mb2-bct-misc-p3767-0000.dts

cd ~/nvidia/Linux_for_Tegra
sudo ./apply_binaries.sh  
sudo ./tools/l4t_flash_prerequisites.sh
sudo ./tools/l4t_create_default_user.sh --accept-license -u ubuntu -p password -a -n jetson1

# Let's modify the kernel before we flash the board.

cd ~/nvidia/Linux_for_Tegra/source
./source_sync.sh -k -t jetson_$JETSON_VERSION

cd ~/nvidia/Linux_for_Tegra/source/kernel
sudo vim ~/nvidia/Linux_for_Tegra/source/kernel/Makefile
  29gg
  ... defconfig
        $(MAKE) \
                ARCH=arm64 \
                -C $(kernel_source_dir) $(O_OPT) \
                LOCALVERSION=$(version) \
                nconfig
  ... Image
  
# Probably smart to take a backup at this point
sudo cp -rp ~/nvidia/Linux_for_Tegra ~/nvidia/Linux_for_Tegra_SYNCED

export CROSS_COMPILE=aarch64-linux-gnu-
make kernel
  # Dig through the configuration to enable the following features and modules. F8 to search.
  #
  # CONFIG_NET_CLS_BPF=m
  # CONFIG_BLK_DEV_THROTTLING=y
  # CONFIG_NET_CLS_CGROUP=m
  # CONFIG_CGROUP_NET_PRIO=y
  # CONFIG_IP_SET=m
  # CONFIG_IP_VS_NFCT=y
  # CONFIG_IP_VS_PROTO_TCP=y
  # CONFIG_IP_VS_PROTO_UDP=y
  # CONFIG_IP_VS_RR=m
  # CONFIG_CRYPTO_SEQIV=m
  # CONFIG_XFRM_USER=y
  # CONFIG_INET_ESP=m
  # CONFIG_NETFILTER_XT_TARGET_TPROXY=m
  # CONFIG_NETFILTER_XT_MATCH_MARK=m
  # CONFIG_NETFILTER_XT_MATCH_SOCKET=m
export INSTALL_MOD_PATH=~/nvidia/Linux_for_Tegra/rootfs/
sudo -E make install
cp ~/nvidia/Linux_for_Tegra/source/kernel/kernel-jammy-src/arch/arm64/boot/Image ~/nvidia/Linux_for_Tegra/kernel/Image

# Install OOT modules
cd ~/nvidia/Linux_for_Tegra
sudo ./tools/l4t_update_initrd.sh

# Connect shit-box to turing-pi via usb
# From turing-pi

tpi power off
tpi usb flash --node 4
tpi power on

# From shit-box

lsusb

sudo ./tools/kernel_flash/l4t_initrd_flash.sh --external-device nvme0n1p1 \
  -c tools/kernel_flash/flash_l4t_external.xml -p "-c bootloader/generic/cfg/flash_t234_qspi.xml"  \
  --showlogs --network usb0 --erase-all --flash-only --keep jetson-orin-nano-devkit internal

ssh-copy-id ubuntu@xxx.xx.x.x
ssh ubuntu@xxx.xx.x.x

# From jetson1

sudo apt update
sudo apt -y upgrade
sudo shutdown -h now

# From turing-pi

tpi power off --node 4
tpi usb device --node 4
tpi power on --node 4

Follow-ups

• NVIDIA Container Runtime

  We're not actually installing Docker, but we still need to ensure the container runtime is available.

  distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
  && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
  && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
        sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
        sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
  
  sudo apt update
  sudo apt install -y nvidia-container-toolkit runc
  
  sudo reboot
  
  # # # #
  # Verify
  # # # #
  
  nvidia-smi
  
  nvidia-container-runtime -v

• jtop

  sudo apt update
  sudo apt install -y python3-pip
  
  sudo pip3 install -U jetson-stats
  
  sudo reboot
  
  # # # #
  # Verify
  # # # #
  
  jtop

• NVIDIA CUDA

  sudo apt-key del 7fa2af80
  wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/arm64/cuda-keyring_1.1-1_all.deb
  sudo dpkg -i cuda-keyring_1.1-1_all.deb
  
  sudo apt update
  sudo apt-get install cuda-toolkit
  sudo apt-get install nvidia-gds
  
  echo 'PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' >> ~/.profile
  echo 'LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' >> ~/.profile
  
  sudo reboot
  
  # # # #
  # Verify
  # # # #
  
  sudo jetson_release -v
  
  cd ~/nvidia
  git clone https://github.com/NVIDIA/cuda-samples.git
  cd cuda-samples/Samples/1_Utilities/deviceQuery
  sudo make
  
  ./deviceQuery

Validate Kernel flags

REQUIRED_CONFIGS=(
  CONFIG_BPF
  CONFIG_BPF_SYSCALL
  CONFIG_NET_CLS_BPF
  CONFIG_BPF_JIT
  CONFIG_NET_CLS_ACT
  CONFIG_NET_SCH_INGRESS
  CONFIG_CRYPTO_SHA1
  CONFIG_CRYPTO_USER_API_HASH
  CONFIG_CGROUPS
  CONFIG_CGROUP_BPF
  CONFIG_PERF_EVENTS
  CONFIG_SCHEDSTATS
  CONFIG_NETFILTER_XT_TARGET_TPROXY
  CONFIG_NETFILTER_XT_TARGET_CT
  CONFIG_NETFILTER_XT_MATCH_MARK
  CONFIG_NETFILTER_XT_MATCH_SOCKET
  CONFIG_BLK_DEV_THROTTLING
  CONFIG_NET_CLS_CGROUP
  CONFIG_CGROUP_NET_PRIO
  CONFIG_IP_SET
  CONFIG_IP_VS_NFCT
  CONFIG_IP_VS_PROTO_TCP
  CONFIG_IP_VS_PROTO_UDP
  CONFIG_IP_VS_RR
  CONFIG_CRYPTO_SEQIV
  CONFIG_XFRM_USER
  CONFIG_INET_ESP
)

for config in "${REQUIRED_CONFIGS[@]}"
do
    zcat /proc/config.gz | grep "${config}[ =]"
done