Configuring IB SRIOV

# STEP - 1 Enabling sriov on PFs
# Use following commands to find the path to enable sriov
lspci  -nn | grep -i "Mellanox"
find /sys -name sriov_numvfs

# Enable SRIOV from the above path for (e.g)
# echo "2" > /sys/devices/pci0000:85/0000:85:00.0/0000:86:00.0/0000:87:00.0/0000:88:00.0/sriov_numvfs
# echo "2" > /sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.0/0000:b0:00.0/0000:b1:00.0/sriov_numvfs
# Above will configure two VFs on each IB device

# Confirm additional ib devices are present on the host
ip link show
lspci  -Dnn | grep -i "Mellanox" | grep "Virtual Function"


# STEP - 2 Configure Node GUID, Port GUID, bind and unbind
# For each VF to be usable, each device must be configured appropriately.
# Following is the configuration for VF0 on PF0. Random node guid and port guid is 
# assigned for this VF. Last two commands have to be run based on information from 
# lspci command from above
echo Follow > /sys/class/infiniband/mlx5_0/device/sriov/0/policy
echo 11:22:33:44:77:66:77:90 > /sys/class/infiniband/mlx5_0/device/sriov/0/node
echo 11:22:33:44:77:66:77:91 > /sys/class/infiniband/mlx5_0/device/sriov/0/port
echo 0000:88:00.1 > /sys/bus/pci/drivers/mlx5_core/unbind
echo 0000:88:00.1 > /sys/bus/pci/drivers/mlx5_core/bind 

# REPEAT above by changing node guid and port guid for additional VFs
# echo Follow > /sys/class/infiniband/mlx5_0/device/sriov/1/policy
# ...
# echo Follow > /sys/class/infiniband/mlx5_1/device/sriov/0/policy
# ...
# echo Follow > /sys/class/infiniband/mlx5_1/device/sriov/1/policy


# STEP - 3 Configure sriov configmap that will be used by the device plugin
# Following is an example that uses device id of 1018. This MUST match the 
# device ids for the VF devices (lspci command above). 
# 1018 is for VF devices created through 
# "Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function]"
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: sriovdp-config
  namespace: kube-system
data:
  config.json: |
    {
        "resourceList": [{
                "resourcePrefix": "mellanox.com",
                "resourceName": "mlnx_sriov_rdma_ib",
                "selectors": {
                    "isRdma": true,
                    "vendors": ["15b3"],
                    "devices": ["1018"],
                    "drivers": ["mlx5_core"]
                }
            }
        ]
    }
EOF



# STEP - 4 Deploy sriov device plugin
kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml

# Wait for pods to be running and run kubectl describe node to verify that new resources are shown as Allocatable
# For (e.g)
# mellanox.com/mlnx_sriov_rdma_ib:       4


# STEP - 5 Deploy ib-sriov cni
kubectl apply -f https://raw.githubusercontent.com/openshift/ib-sriov-cni/master/deployment/ib-sriov-cni-daemonset.yaml


# STEP - 6 Multus configuration to use ib-sriov cni
cat <<EOF | kubectl apply -f -
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: ib-sriov-network
  annotations:
    k8s.v1.cni.cncf.io/resourceName: mellanox.com/mlnx_sriov_rdma_ib
spec:
  config: '{
  "type": "ib-sriov",
  "cniVersion": "0.3.1",
  "name": "sriov-network",
  "pkey": "0xffff",
  "link_state": "enable",
  "ibKubernetesEnabled": false,
  "ipam": {
                    "datastore": "kubernetes",
                    "kubernetes": {
                        "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig"
                    },
                    "log_file": "/tmp/whereabouts.log",
                    "log_level": "debug",
                    "range": "192.168.119.0/24",
                    "type": "whereabouts"
  }
}'
EOF


# STEP - 7 Deploy pod with IB device resource that also uses ib-sriov Multus NAD
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: sriov-pod
  annotations:
    k8s.v1.cni.cncf.io/networks: '[{ "name": "ib-sriov-network", "interface" : "ib0" }]'
spec:
  restartPolicy: OnFailure
  containers:
  - image: vitaliyrazinkov/ubuntu-ofed-mlnx:18.04
    name: sriov-pod
    securityContext:
      capabilities:
        add: [ "IPC_LOCK", "SYS_ADMIN", "NET_ADMIN", "NET_RAW" ]
    resources:
      limits:
        mellanox.com/mlnx_sriov_rdma_ib: 1
        cpu: "1"
    command:
    - sh
    - -c
    - |
      ls -l /dev/infiniband /sys/class/net
      sleep inf
 EOF