Skip to content

Instantly share code, notes, and snippets.

@jacob-delgado

jacob-delgado/kind-ovn-multus-istio.log

Forked from s1061123/kind-multus-test.log
Last active June 13, 2023 23:39
Show Gist options
  • Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.
Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.
Download ZIP
Kind with Multus log
Raw
kind-ovn-multus-istio.log
## Create 4-node environment config
#$ cat << EOF > config-4node.yml
#kind: Cluster
#apiVersion: kind.x-k8s.io/v1alpha4
#nodes:
# - role: control-plane
# - role: worker
# - role: worker
# - role: worker
# EOF
## Create cluster
# $ kind create cluster --config config-4node.yml
# clone ovn
$ git clone https://github.com/ovn-org/ovn-kubernetes
$ cd ovn-kubernetes
$ cd contrib
$ ./kind.sh -wk 3
$ export KUBECONFIG=${HOME}/ovn.conf
## get nodes
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
ovn-control-plane Ready control-plane 6m41s v1.26.0
ovn-worker Ready <none> 6m10s v1.26.0
ovn-worker2 Ready <none> 6m10s v1.26.0
ovn-worker3 Ready <none> 6m10s v1.26.0
## install multus
$ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.2/deployments/multus-daemonset.yml
## get koko
$ curl -LO https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
$ chmod +x koko_0.83_linux_amd64
## Create veth interface between ovn-woker and ovn-worker2
$ sudo ./koko_0.83_linux_amd64 -d ovn-worker,eth1 -d ovn-worker2,eth1
# install cni reference plugins (kindnet doesn't install ipvlan or macvlan)
$ cat << EOF > cni-install.yml
---
kind: ConfigMap
apiVersion: v1
metadata:
name: cni-install-sh
namespace: kube-system
data:
install_cni.sh: |
cd /tmp
wget https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
cd /host/opt/cni/bin
tar xvfzp /tmp/cni-plugins-linux-amd64-v1.3.0.tgz
sleep infinite
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: install-cni-plugins
namespace: kube-system
labels:
name: cni-plugins
spec:
selector:
matchLabels:
name: cni-plugins
template:
metadata:
labels:
name: cni-plugins
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/arch: amd64
tolerations:
- operator: Exists
effect: NoSchedule
containers:
- name: install-cni-plugins
image: alpine
command: ["/bin/sh", "/scripts/install_cni.sh"]
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: true
volumeMounts:
- name: cni-bin
mountPath: /host/opt/cni/bin
- name: scripts
mountPath: /scripts
volumes:
- name: cni-bin
hostPath:
path: /opt/cni/bin
- name: scripts
configMap:
name: cni-install-sh
items:
- key: install_cni.sh
path: install_cni.sh
EOF
$ kubectl apply -f cni-install.yml
## create macvlan
$ cat << EOF > macvlan.yml
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: macvlan-conf
spec:
config: '{
"cniVersion": "0.3.1",
"plugins": [
{
"type": "macvlan",
"capabilities": { "ips": true },
"master": "eth1",
"mode": "bridge",
"ipam": {
"type": "static"
}
}, {
"type": "tuning"
} ]
}'
EOF
$ kubectl apply -f macvlan.yml
# install istio
$ helm repo add istio https://istio-release.storage.googleapis.com/charts
$ helm repo update
$ kubectl create namespace istio-system
$ cat << EOF > overrides.yml
istio_cni:
enabled: true
chained: false
cni:
enabled: true
chained: false
cniBinDir: /opt/cni/bin
cniConfDir: /etc/cni/multus/net.d
cniConfFileName: istio-cni.conf
excludeNamespaces:
- istio-system
- kube-system
EOF
$ helm install istio-base istio/base -n istio-system --version 1.17.2
$ helm install istio-cni istio/cni --namespace kube-system --wait --values overrides.yml --version 1.17.2
$ helm install istiod istio/istiod -n istio-system --wait --values overrides.yml --version 1.17.2
# istio-cni network-attachment-definition
$ cat <<EOF > istio-cni.yaml
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: istio-cni
spec:
config: ''
EOF
$ kubectl apply -f istio-cni.yaml
$ kubectl label namespace default istio-injection=enabled --overwrite
# httpbin-multus
# apply httpbin w/nodeSelector = ovn-worker
$ cat <<EOF > httpbin-multus-default.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: httpbin
---
apiVersion: v1
kind: Service
metadata:
name: httpbin
labels:
app: httpbin
service: httpbin
spec:
ports:
- name: http
port: 8000
targetPort: 80
selector:
app: httpbin
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpbin
spec:
replicas: 1
selector:
matchLabels:
app: httpbin
version: v1
template:
metadata:
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"ips": ["10.1.1.11/24"] }
]'
labels:
app: httpbin
version: v1
spec:
nodeSelector:
kubernetes.io/hostname: ovn-worker
serviceAccountName: httpbin
containers:
- image: docker.io/kennethreitz/httpbin
imagePullPolicy: IfNotPresent
name: httpbin
ports:
- containerPort: 80
EOF
$ kubectl apply -f httpbin-multus-default.yaml
# sleep-multus
# apply sleep w/nodeSelector = ovn-worker2
$ cat <<EOF > sleep-multus-default.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: sleep-multus
---
apiVersion: v1
kind: Service
metadata:
name: sleep-multus
labels:
app: sleep-multus
service: sleep-multus
spec:
ports:
- port: 80
name: http
selector:
app: sleep-multus
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sleep-multus
spec:
replicas: 1
selector:
matchLabels:
app: sleep-multus
template:
metadata:
labels:
app: sleep-multus
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"ips": ["10.1.1.12/24"] }
]'
spec:
nodeSelector:
kubernetes.io/hostname: ovn-worker2
terminationGracePeriodSeconds: 0
serviceAccountName: sleep-multus
containers:
- name: sleep-multus
image: curlimages/curl
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /etc/sleep/tls
name: secret-volume
volumes:
- name: secret-volume
secret:
secretName: sleep-multus-secret
optional: true
EOF
$ kubectl apply -f sleep-multus-default.yaml
# sleep
# not a part of the multus macvlan-conf network
$ cat <<EOF > sleep-nomultus-default.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: sleep
---
apiVersion: v1
kind: Service
metadata:
name: sleep
labels:
app: sleep
service: sleep
spec:
ports:
- port: 80
name: http
selector:
app: sleep
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sleep
spec:
replicas: 1
selector:
matchLabels:
app: sleep
template:
metadata:
labels:
app: sleep
spec:
terminationGracePeriodSeconds: 0
serviceAccountName: sleep
containers:
- name: sleep
image: curlimages/curl
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /etc/sleep/tls
name: secret-volume
volumes:
- name: secret-volume
secret:
secretName: sleep-secret
optional: true
EOF
$ kubectl apply -f sleep-nomultus-default.yaml
$ kubectl create ns nosidecar
# sleep in nosidecar namespace on macvlan-conf multus network
$ cat <<EOF > sleep-httpbin-nomultus-nosidecar.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: sleep
---
apiVersion: v1
kind: Service
metadata:
name: sleep
labels:
app: sleep
service: sleep
spec:
ports:
- port: 80
name: http
selector:
app: sleep
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sleep
spec:
replicas: 1
selector:
matchLabels:
app: sleep
template:
metadata:
labels:
app: sleep
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"namespace": "default",
"ips": ["10.1.1.13/24"] }
]'
spec:
nodeSelector:
kubernetes.io/hostname: ovn-worker2
terminationGracePeriodSeconds: 0
serviceAccountName: sleep
containers:
- name: sleep
image: curlimages/curl
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /etc/sleep/tls
name: secret-volume
volumes:
- name: secret-volume
secret:
secretName: sleep-secret
optional: true
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: httpbin
---
apiVersion: v1
kind: Service
metadata:
name: httpbin
labels:
app: httpbin
service: httpbin
spec:
ports:
- name: http
port: 8000
targetPort: 80
selector:
app: httpbin
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpbin
spec:
replicas: 1
selector:
matchLabels:
app: httpbin
version: v1
template:
metadata:
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"namespace": "default",
"ips": ["10.1.1.14/24"] }
]'
labels:
app: httpbin
version: v1
spec:
nodeSelector:
kubernetes.io/hostname: ovn-worker
serviceAccountName: httpbin
containers:
- image: docker.io/kennethreitz/httpbin
imagePullPolicy: IfNotPresent
name: httpbin
ports:
- containerPort: 80
EOF
$ kubectl apply -f sleep-httpbin-nomultus-nosidecar.yaml -n nosidecar
$ kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default httpbin ClusterIP 10.96.92.191 <none> 8000/TCP 32m
default kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 49m
default sleep ClusterIP 10.96.164.183 <none> 80/TCP 29m
default sleep-multus ClusterIP 10.96.63.157 <none> 80/TCP 30m
istio-system istiod ClusterIP 10.96.136.7 <none> 15010/TCP,15012/TCP,443/TCP,15014/TCP 38m
kube-system kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 49m
nosidecar httpbin ClusterIP 10.96.71.12 <none> 8000/TCP 27m
nosidecar sleep ClusterIP 10.96.168.196 <none> 80/TCP 27m
ovn-kubernetes ovnkube-db ClusterIP None <none> 6641/TCP,6642/TCP 47m
# multus network topology
# default namespace | IP address | node
# httpbin | 10.1.1.11 | ovn-worker
# sleep-multus | 10.1.1.12 | ovn-worker2
# sleep | 10.1.1.13 | ovn-worker2
# nosidecar namespace
# httpbin | 10.1.1.14 | ovn-worker
$ export SLEEP_MULTUS_DEFAULT_POD=$(kubectl get pod -l app=sleep-multus -o jsonpath={.items..metadata.name})
$ export SLEEP_DEFAULT_POD=$(kubectl get pod -l app=sleep -o jsonpath={.items..metadata.name})
$ export SLEEP_NOSIDECAR_POD=$(kubectl get pod -n nosidecar -l app=sleep -o jsonpath={.items..metadata.name})
$ kubectl exec -it $SLEEP_MULTUS_DEFAULT_POD -- curl http://10.1.1.11/get
{
"args": {},
"headers": {
"Accept": "*/*",
"Host": "10.1.1.11",
"User-Agent": "curl/8.1.1-DEV",
"X-B3-Parentspanid": "584b291281342c63",
"X-B3-Sampled": "0",
"X-B3-Spanid": "8919de5bdc0fc795",
"X-B3-Traceid": "176d409680c7ba84584b291281342c63",
"X-Envoy-Attempt-Count": "1"
},
"origin": "127.0.0.6",
"url": "http://10.1.1.11/get"
}
$ kubectl exec -it $SLEEP_DEFAULT_POD -- curl http://10.1.1.11/get
This should hang.
kubectl exec -it $SLEEP_NOSIDECAR_POD -n nosidecar -- curl http://10.1.1.11/get 1 ↵
{
"args": {},
"headers": {
"Accept": "*/*",
"Host": "10.1.1.11",
"User-Agent": "curl/8.1.1-DEV",
"X-B3-Sampled": "0",
"X-B3-Spanid": "4d0eb020ac5c6ce4",
"X-B3-Traceid": "a07130fe440fefd74d0eb020ac5c6ce4"
},
"origin": "127.0.0.6",
"url": "http://10.1.1.11/get"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment