sriov in VM with kube-1.16

install-k8s.sh

#!/bin/bash
set -x

## On the undercloud, don't forget to configure the DNS

yum -y install docker
systemctl enable docker
systemctl start docker

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF

# Set SELinux in permissive mode (effectively disabling it)
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

#kubelet requires swap to be disabled!
swapoff -a
sed -i.bak '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes 

systemctl enable kubelet && systemctl start kubelet

cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
sysctl --system

kubeadm init --pod-network-cidr=10.244.0.0/16
mkdir -p /$USER/.kube && cp /etc/kubernetes/admin.conf /$USER/.kube/config

wait_for_running_pods() {
# $1: number of Running pods to wait for (at least)
if [ X$2 == X ] ; then 
NS="--all-namespaces"
else
NS="-n $2"
fi
I=0
LOOP=0
while [ `kubectl get pods $NS | grep Running | wc -l ` -lt $1 ] ; do
  sleep 10
  echo -n .
  I=$((I+1))
  if [ $I -lt 10 ] ; then
    I=0
    kubectl get pods $NS 
  fi
  LOOP=$((LOOP+1))
  if [ $LOOP -gt 12 ] ; then
    exit
  fi
done
kubectl get pods $NS 
}

# coredns pods won't start until flannel is started
#
# kube-system   coredns-86c58d9df4-gtwm4               0/1     Pending   0          2m46s
# kube-system   coredns-86c58d9df4-j9sxt               0/1     Pending   0          2m46s
# kube-system   etcd-computeocp-0                      1/1     Running   0          114s
# kube-system   kube-apiserver-computeocp-0            1/1     Running   0          107s
# kube-system   kube-controller-manager-computeocp-0   1/1     Running   0          110s
# kube-system   kube-proxy-6ssts                       1/1     Running   0          2m46s
# kube-system   kube-scheduler-computeocp-0            1/1     Running   0          2m5s
wait_for_running_pods 5	

# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

kubectl taint nodes --all node-role.kubernetes.io/master-

wait_for_running_pods 8	

echo "kubernbetes installed, you can proceed with multus and sriov"

install-multus-device-plugin-in-vm.sh

#!/bin/bash
set -x

if [ -f /root/PCI_ID.env ] ; then
	source /root/PCI_ID.env
else
	RADIO_PCI=00:09.0
	UPLINK_PCI=00:04.0
fi

wait_for_running_pods() {
# $1: number of Running pods to wait for (at least)
if [ X$2 == X ] ; then
NS="--all-namespaces"
else
NS="-n $2"
fi
I=0
LOOP=0
while [ `kubectl get pods $NS | grep Running | wc -l ` -lt $1 ] ; do
  sleep 10
  echo -n .
  I=$((I+1))
  if [ $I -lt 10 ] ; then
    I=0
    kubectl get pods $NS
  fi
  LOOP=$((LOOP+1))
  if [ $LOOP -gt 12 ] ; then
    exit
  fi
done
kubectl get pods $NS
}

########
# multus
########
wget https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset.yml
kubectl apply -f multus-daemonset.yml

cd /etc/cni/net.d/
rm -f 00-multus.conf
cat << EOF > 05-multus.conf
{
  "name": "multus-cni-network",
  "type": "multus",
  "cniVersion": "0.3.1",
  "delegates": [
    {
      "type": "flannel",
      "name": "flannel.1",
      "delegate": {
        "isDefaultGateway": true,
        "hairpinMode": true
      }
    }
  ],
  "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
  "logFile": "/var/log/multus.log",
  "logLevel": "debug"
}
EOF
cd -

#######
# SRIOV
#######

# Device plugin
#yum install -y driverctl
#echo "options vfio enable_unsafe_noiommu_mode=1" > /etc/modprobe.d/vfio-noiommu.conf
#driverctl set-override 0000:$RADIO_PCI vfio-pci
#driverctl set-override 0000:$UPLINK_PCI vfio-pci
#driverctl list-overrides

cat << EOF > sriovdp.yml 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: sriov-nodes-config
  namespace: kube-system
data:
  fallback-config: |
    {
        "resourceList":
        [
            {
                "resourceName": "radio_net",
                "rootDevices": ["$RADIO_PCI"],
                "sriovMode": false,
                "deviceType": "vfio"
            },
            {
                "resourceName": "uplink_net",
                "rootDevices": ["$UPLINK_PCI"],
                "sriovMode": false,
                "deviceType": "vfio"
            }
        ]
    }
---	
apiVersion: v1
kind: ServiceAccount
metadata:
  name: sriov-device-plugin
  namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: sriov-device-plugin
  namespace: kube-system
  labels:
    tier: node
    app: sriovdp
spec:
  selector:
    matchLabels:
      name: sriov-device-plugin
  template:
    metadata:
      labels:
        name: sriov-device-plugin
        tier: node
        app: sriovdp
    spec:
      nodeSelector:
        beta.kubernetes.io/arch: amd64
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      serviceAccountName: sriov-device-plugin
      initContainers:
      - name: config-container
        image: docker.io/nfvpe/sriov-device-plugin:v2.1.0
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        command:
        - /bin/sh
        - -c
        args:
        - |
          if [ -f /etc/sriov-nodes-config/\$NODE_NAME ]; then
            config_path=/etc/sriov-nodes-config/\$NODE_NAME
          else
            config_path=/etc/sriov-nodes-config/fallback-config
          fi
          cp \$config_path /etc/pcidp/config.json
        volumeMounts:
        - name: config
          mountPath: /etc/pcidp
        - name: configmap
          mountPath: /etc/sriov-nodes-config
        securityContext:
          privileged: true
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
      containers:
      - name: kube-sriovdp
        image: docker.io/nfvpe/sriov-device-plugin:v2.1.0
        args:
        - --log-level=10
        securityContext:
          privileged: false
        volumeMounts:
        - name: devicesock
          mountPath: /var/lib/kubelet/device-plugins/
          readOnly: false
        - name: sysfs
          mountPath: /sys
          readOnly: true
        - name: config
          mountPath: /etc/pcidp
          readOnly: true
      volumes:
        - name: devicesock
          hostPath:
            path: /var/lib/kubelet/device-plugins/
        - name: sysfs
          hostPath:
            path: /sys
        - name: config
          hostPath:
            path: /etc/pcidp
            type: DirectoryOrCreate
        - name: configmap
          configMap:
            name: sriov-nodes-config
EOF
kubectl create -f sriovdp.yml
kubectl get nodes -o json | jq .items[0].status.allocatable

###
# pod testing
###
cat << EOF > pod-testpmd.yml
apiVersion: v1
kind: Pod
metadata:
  name: pod-testpmd
  annotations:
    # br0 is the name of the CRD
    k8s.v1.cni.cncf.io/networks: radio-net,uplink-net
spec:
  hostNetwork: true
  hostPID: true
  hostIPC: true
  containers:
  - name: samplepod
    command: ["sleep", "99999"]
    image: dockerfranck/testpmd
    securityContext:
      privileged: true
    resources:
      requests:
        intel.com/radio_net: "1"
        intel.com/uplink_net: "1"
        memory: 100Mi
        hugepages-1Gi: 1Gi
        cpu: '4'
      limits:
        intel.com/radio_net: "1"
        intel.com/uplink_net: "1"
        hugepages-1Gi: 1Gi
        cpu: '4'
        memory: 100Mi
    volumeMounts:
    - mountPath: /dev/hugepages
      name: hugepage
      readOnly: False
  volumes:
  - name: hugepage
    emptyDir:
      medium: HugePages
EOF

kubectl get nodes -o json | jq .items[0].status.allocatable
kubectl apply -f pod-testpmd.yml
wait_for_running_pods 1 default
kubectl describe pods pod-testpmd