microshift-on-jetson.md

Environment

Jetpack 4.6, Jetson Xavier, Ubuntu 18.04

prepare

apt install -y curl jq runc iptables conntrack nvidia-container-runtime nvidia-container-toolkit

crio

Install crio

curl https://raw.githubusercontent.com/cri-o/cri-o/main/scripts/get | bash 

Config crio

use nvidia-container-runtime

rm /etc/crio/crio.conf.d/*

cat << EOF > /etc/crio/crio.conf.d/10-nvidia-runtime.conf
[crio.runtime]
default_runtime = "nvidia"

[crio.runtime.runtimes.nvidia]
runtime_path = "/usr/bin/nvidia-container-runtime"
EOF

cat << EOF > /etc/crio/crio.conf.d/01-crio-runc.conf
[crio.runtime.runtimes.runc]
runtime_path = "/usr/sbin/runc"
runtime_type = "oci"
runtime_root = "/run/runc"
EOF

cat << EOF > /etc/cni/net.d/100-crio-bridge.conf
{
    "cniVersion": "0.4.0",
    "name": "crio",
    "type": "bridge",
    "bridge": "cni0",
    "isGateway": true,
    "ipMasq": true,
    "hairpinMode": true,
    "ipam": {
        "type": "host-local",
        "routes": [
            { "dst": "0.0.0.0/0" }
        ],
        "ranges": [
            [{ "subnet": "10.42.0.0/24" }]
        ]
    }
}
EOF

Microshift

install

export ARCH=arm64
export VERSION=$(curl -s https://api.github.com/repos/redhat-et/microshift/releases | grep tag_name | head -n 1 | cut -d '"' -f 4)
curl -LO https://github.com/redhat-et/microshift/releases/download/$VERSION/microshift-linux-${ARCH}
mv microshift-linux-${ARCH} /usr/local/bin/microshift; chmod 755 /usr/local/bin/microshift

config microshift

cat << EOF > /usr/lib/systemd/system/microshift.service
[Unit]
Description=MicroShift
After=crio.service

[Service]
WorkingDirectory=/usr/local/bin/
ExecStart=/usr/local/bin/microshift run
Restart=always
User=root

[Install]
WantedBy=multi-user.target
EOF

Enable crio and microshift systemd

systemctl enable crio --now
systemctl enable microshift.service --now

kubectl

Install kubectl

export ARCH=arm64
curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${ARCH}/kubectl" 
chmod +x ./kubectl
mv ./kubectl /usr/local/bin/kubectl 

set kubeconfig

export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig

Test

Microshift

Once microshift is up, these pods should be up:

# kubectl get pods -A
NAMESPACE                       NAME                                  READY   STATUS      RESTARTS   AGE
kube-system                     kube-flannel-ds-rc85f                 1/1     Running     1          3d1h
kubevirt-hostpath-provisioner   kubevirt-hostpath-provisioner-bnrz2   1/1     Running     0          3d1h
openshift-dns                   dns-default-p2cc5                     2/2     Running     0          3d1h
openshift-dns                   node-resolver-wd7mx                   1/1     Running     1          3d1h
openshift-ingress               router-default-85bcfdd948-4nx7k       1/1     Running     0          3d1h
openshift-service-ca            service-ca-76674bfb58-fbp4p           1/1     Running     0          3d1h

GPU workload

run this workload to test GPU

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: nvidia-l4t-base
spec:
  restartPolicy: OnFailure
  containers:
  - name: nvidia-l4t-base
    image: "quay.io/microshift/microshift-aio:l4t-test"
    command:
       - /usr/bin/deviceQuery
EOF

Once the Pod is done, this is the expected log:

# kubectl logs nvidia-l4t-base
/usr/bin/deviceQuery Starting...

 CUDA Device Query (Runtime API) version (CUDART static linking)

Detected 1 CUDA Capable device(s)

Device 0: "Xavier"
  CUDA Driver Version / Runtime Version          10.2 / 10.2
  CUDA Capability Major/Minor version number:    7.2
  ...
  blah blah blah

A full demo is at