Using this dummy CNI script...
Pay attention to the cniresult() routine, which returns... two interfaces.
#!/usr/bin/env bash
# DEBUG=true
# LOGFILE=/tmp/seamless.log
# Outputs errors to stderr
errorlog () {
>&2 echo $1
}
# Logs for debugging.
debuglog () {
# Set DEBUG to anything to enable debug output.
if [ -n "$DEBUG" ]; then
# touch /tmp/seamless_a
# Set logfile if you want to log to a flat file.
if [ -n "$LOGFILE" ]; then
# touch /tmp/seamless_b
echo $1 >> $LOGFILE.$CNI_CONTAINERID
else
# Otherwise, log to stderr.
>&2 echo $1
fi
fi
}
# Outputs an enhanced CNI result with two interfaces and three IP addresses.
cniresult () {
cat << EOF
{
"cniVersion": "1.1.0",
"interfaces": [
{
"name": "example0",
"mac": "00:AA:BB:CC:DD:01",
"mtu": 1500,
"sandbox": "/path/to/network/namespace"
},
{
"name": "example1",
"mac": "00:AA:BB:CC:DD:02",
"mtu": 1500,
"sandbox": "/path/to/network/namespace"
}
],
"ips": [
{
"address": "192.0.2.1/24",
"gateway": "192.0.2.254",
"interface": 0
},
{
"address": "192.0.2.2/24",
"interface": 0
},
{
"address": "192.0.2.3/24",
"gateway": "192.0.2.254",
"interface": 1
}
],
"dns": {
"nameservers": ["8.8.8.8", "8.8.4.4"],
"search": ["example.com"],
"options": ["ndots:2"]
}
}
EOF
}
# Certain failures we want to exit on.
exit_on_error() {
exit_code=$1
last_command=${@:2}
if [ $exit_code -ne 0 ]; then
>&2 echo "dummy: \"${last_command}\" command failed with exit code ${exit_code}."
cniresult
exit $exit_code
fi
}
# Overarching basic parameters.
containerifname=eth0
# --------------------------------------- REFERENCE: Common environment variables.
debuglog "CNI method: $CNI_COMMAND"
debuglog "CNI container id: $CNI_CONTAINERID"
debuglog "CNI netns: $CNI_NETNS"
cniresult
Then I execute it with something like...
---
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: twointerface
spec:
config: '{
"cniVersion": "0.3.1",
"type": "twointerface",
"name": "twointerface-test",
"capabilities": {
"ips": true
}
}'
---
apiVersion: v1
kind: Pod
metadata:
name: debugpod
annotations:
k8s.v1.cni.cncf.io/networks: twointerface
spec:
nodeSelector:
multusdebug: "true"
containers:
- name: debugcontainer
command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
image: alpine
---
And that pod doesn't represent multiple interfaces...
[fedora@labkube-master-1 ~]$ kubectl describe pod debugpod
Name: debugpod
Namespace: default
Priority: 0
Service Account: default
Node: labkube-node-1/192.168.122.245
Start Time: Thu, 25 Jul 2024 04:19:12 +0900
Labels: <none>
Annotations: k8s.v1.cni.cncf.io/network-status:
[{
"name": "cbr0",
"interface": "eth0",
"ips": [
"10.244.2.3"
],
"mac": "c2:d4:1d:99:1e:14",
"default": true,
"dns": {},
"gateway": [
"10.244.2.1"
]
},{
"name": "default/twointerface",
"interface": "example1",
"ips": [
"192.0.2.1",
"192.0.2.2",
"192.0.2.3"
],
"mac": "00:AA:BB:CC:DD:02",
"mtu": 1500,
"dns": {
"nameservers": [
"8.8.8.8",
"8.8.4.4"
],
"search": [
"example.com"
],
"options": [
"ndots:2"
]
}
}]
k8s.v1.cni.cncf.io/networks: twointerface
Take a look at this result from bridge CNI, it results in multiple interfaces, but, do we want to know about them all?
[fedora@labkube-master-1 ~]$ sudo CNI_PATH=/opt/cni/bin /home/fedora/go/bin/cnitool add mynet /var/run/netns/testing
{
"cniVersion": "0.3.1",
"interfaces": [
{
"name": "mynet0",
"mac": "5e:62:bb:d0:e8:91"
},
{
"name": "vethdfff5875",
"mac": "0e:34:c9:64:ad:2e"
},
{
"name": "eth0",
"mac": "72:23:ac:b2:46:9b",
"sandbox": "/var/run/netns/testing"
}
],
"ips": [
{
"version": "4",
"interface": 2,
"address": "10.10.0.2/16",
"gateway": "10.10.0.1"
}
],
"routes": [
{
"dst": "0.0.0.0/0",
"gw": "10.10.0.1"
}
],
"dns": {}
}[fedora@labkube-master-1 ~]$ cat /etc/cni/net.d/99-bridge.conf
{
"cniVersion": "0.3.1",
"name": "mynet",
"type": "bridge",
"bridge": "mynet0",
"isDefaultGateway": true,
"forceAddress": false,
"ipMasq": true,
"hairpinMode": true,
"ipam": {
"type": "host-local",
"subnet": "10.10.0.0/16"
}
}
Deploy Multus, with a modified https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick.yml:
[fedora@labkube-master-1 ~]$ diff multus-daemonset-thick.yml multus-daemonset-thick.mod.yml
157c157
< image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
---
> image: quay.io/dosmith/multus-cni:networkstatuses_b
202c202
< image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
---
> image: quay.io/dosmith/multus-cni:networkstatuses_b
Then, with the same labeled node and dummy CNI plugin...
[fedora@labkube-master-1 ~]$ cat test.yaml
---
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: twointerface
spec:
config: '{
"cniVersion": "0.3.1",
"type": "twointerface",
"name": "twointerface-test",
"capabilities": {
"ips": true
}
}'
---
apiVersion: v1
kind: Pod
metadata:
name: debugpod
annotations:
k8s.v1.cni.cncf.io/networks: twointerface
spec:
nodeSelector:
multusdebug: "true"
containers:
- name: debugcontainer
command: ["/bin/ash", "-c", "trap : TERM INT; sleep infinity & wait"]
image: alpine
---
[fedora@labkube-master-1 ~]$
[fedora@labkube-master-1 ~]$
[fedora@labkube-master-1 ~]$
[fedora@labkube-master-1 ~]$ kubectl create -f test.yaml
networkattachmentdefinition.k8s.cni.cncf.io/twointerface created
pod/debugpod created
[fedora@labkube-master-1 ~]$ watch -n2 kubectl get pods -A -o wide
[fedora@labkube-master-1 ~]$ kubectl describe pod debugpod
Name: debugpod
Namespace: default
Priority: 0
Service Account: default
Node: labkube-node-1/192.168.122.245
Start Time: Fri, 26 Jul 2024 04:45:36 +0900
Labels: <none>
Annotations: k8s.v1.cni.cncf.io/network-status:
[{
"name": "cbr0",
"interface": "eth0",
"ips": [
"10.244.2.4"
],
"mac": "c6:ca:e3:10:65:66",
"default": true,
"dns": {},
"gateway": [
"10.244.2.1"
]
},{
"name": "default/twointerface",
"interface": "example0",
"ips": [
"192.0.2.1",
"192.0.2.2"
],
"mac": "00:AA:BB:CC:DD:01",
"mtu": 1500,
"dns": {
"nameservers": [
"8.8.8.8",
"8.8.4.4"
],
"search": [
"example.com"
],
"options": [
"ndots:2"
]
}
},{
"name": "default/twointerface",
"interface": "example1",
"ips": [
"192.0.2.3"
],
"mac": "00:AA:BB:CC:DD:02",
"mtu": 1500,
"dns": {
"nameservers": [
"8.8.8.8",
"8.8.4.4"
],
"search": [
"example.com"
],
"options": [
"ndots:2"
]
}
}]
We can see that we properly get two interfaces.