-
-
Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.
Kind with Multus log
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
## Create 4-node environment config | |
#$ cat << EOF > config-4node.yml | |
#kind: Cluster | |
#apiVersion: kind.x-k8s.io/v1alpha4 | |
#nodes: | |
# - role: control-plane | |
# - role: worker | |
# - role: worker | |
# - role: worker | |
# EOF | |
## Create cluster | |
# $ kind create cluster --config config-4node.yml | |
# clone ovn | |
$ git clone https://github.com/ovn-org/ovn-kubernetes | |
$ cd ovn-kubernetes | |
$ cd contrib | |
$ ./kind.sh -wk 3 | |
$ export KUBECONFIG=${HOME}/ovn.conf | |
## get nodes | |
$ kubectl get nodes | |
NAME STATUS ROLES AGE VERSION | |
ovn-control-plane Ready control-plane 6m41s v1.26.0 | |
ovn-worker Ready <none> 6m10s v1.26.0 | |
ovn-worker2 Ready <none> 6m10s v1.26.0 | |
ovn-worker3 Ready <none> 6m10s v1.26.0 | |
## install multus | |
$ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.2/deployments/multus-daemonset.yml | |
## get koko | |
$ curl -LO https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64 | |
$ chmod +x koko_0.83_linux_amd64 | |
## Create veth interface between ovn-woker and ovn-worker2 | |
$ sudo ./koko_0.83_linux_amd64 -d ovn-worker,eth1 -d ovn-worker2,eth1 | |
# install cni reference plugins (kindnet doesn't install ipvlan or macvlan) | |
$ cat << EOF > cni-install.yml | |
--- | |
kind: ConfigMap | |
apiVersion: v1 | |
metadata: | |
name: cni-install-sh | |
namespace: kube-system | |
data: | |
install_cni.sh: | | |
cd /tmp | |
wget https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz | |
cd /host/opt/cni/bin | |
tar xvfzp /tmp/cni-plugins-linux-amd64-v1.3.0.tgz | |
sleep infinite | |
--- | |
apiVersion: apps/v1 | |
kind: DaemonSet | |
metadata: | |
name: install-cni-plugins | |
namespace: kube-system | |
labels: | |
name: cni-plugins | |
spec: | |
selector: | |
matchLabels: | |
name: cni-plugins | |
template: | |
metadata: | |
labels: | |
name: cni-plugins | |
spec: | |
hostNetwork: true | |
nodeSelector: | |
kubernetes.io/arch: amd64 | |
tolerations: | |
- operator: Exists | |
effect: NoSchedule | |
containers: | |
- name: install-cni-plugins | |
image: alpine | |
command: ["/bin/sh", "/scripts/install_cni.sh"] | |
resources: | |
requests: | |
cpu: "100m" | |
memory: "50Mi" | |
limits: | |
cpu: "100m" | |
memory: "50Mi" | |
securityContext: | |
privileged: true | |
volumeMounts: | |
- name: cni-bin | |
mountPath: /host/opt/cni/bin | |
- name: scripts | |
mountPath: /scripts | |
volumes: | |
- name: cni-bin | |
hostPath: | |
path: /opt/cni/bin | |
- name: scripts | |
configMap: | |
name: cni-install-sh | |
items: | |
- key: install_cni.sh | |
path: install_cni.sh | |
EOF | |
$ kubectl apply -f cni-install.yml | |
## create macvlan | |
$ cat << EOF > macvlan.yml | |
apiVersion: "k8s.cni.cncf.io/v1" | |
kind: NetworkAttachmentDefinition | |
metadata: | |
name: macvlan-conf | |
spec: | |
config: '{ | |
"cniVersion": "0.3.1", | |
"plugins": [ | |
{ | |
"type": "macvlan", | |
"capabilities": { "ips": true }, | |
"master": "eth1", | |
"mode": "bridge", | |
"ipam": { | |
"type": "static" | |
} | |
}, { | |
"type": "tuning" | |
} ] | |
}' | |
EOF | |
$ kubectl apply -f macvlan.yml | |
# install istio | |
$ helm repo add istio https://istio-release.storage.googleapis.com/charts | |
$ helm repo update | |
$ kubectl create namespace istio-system | |
$ cat << EOF > overrides.yml | |
istio_cni: | |
enabled: true | |
chained: false | |
cni: | |
enabled: true | |
chained: false | |
cniBinDir: /opt/cni/bin | |
cniConfDir: /etc/cni/multus/net.d | |
cniConfFileName: istio-cni.conf | |
excludeNamespaces: | |
- istio-system | |
- kube-system | |
EOF | |
$ helm install istio-base istio/base -n istio-system --version 1.17.2 | |
$ helm install istio-cni istio/cni --namespace kube-system --wait --values overrides.yml --version 1.17.2 | |
$ helm install istiod istio/istiod -n istio-system --wait --values overrides.yml --version 1.17.2 | |
# istio-cni network-attachment-definition | |
$ cat <<EOF > istio-cni.yaml | |
apiVersion: "k8s.cni.cncf.io/v1" | |
kind: NetworkAttachmentDefinition | |
metadata: | |
name: istio-cni | |
spec: | |
config: '' | |
EOF | |
$ kubectl apply -f istio-cni.yaml | |
$ kubectl label namespace default istio-injection=enabled --overwrite | |
# httpbin-multus | |
# apply httpbin w/nodeSelector = ovn-worker | |
$ cat <<EOF > httpbin-multus-default.yaml | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: httpbin | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: httpbin | |
labels: | |
app: httpbin | |
service: httpbin | |
spec: | |
ports: | |
- name: http | |
port: 8000 | |
targetPort: 80 | |
selector: | |
app: httpbin | |
--- | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: httpbin | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: httpbin | |
version: v1 | |
template: | |
metadata: | |
annotations: | |
k8s.v1.cni.cncf.io/networks: '[ | |
{ "name": "macvlan-conf", | |
"ips": ["10.1.1.11/24"] } | |
]' | |
labels: | |
app: httpbin | |
version: v1 | |
spec: | |
nodeSelector: | |
kubernetes.io/hostname: ovn-worker | |
serviceAccountName: httpbin | |
containers: | |
- image: docker.io/kennethreitz/httpbin | |
imagePullPolicy: IfNotPresent | |
name: httpbin | |
ports: | |
- containerPort: 80 | |
EOF | |
$ kubectl apply -f httpbin-multus-default.yaml | |
# sleep-multus | |
# apply sleep w/nodeSelector = ovn-worker2 | |
$ cat <<EOF > sleep-multus-default.yaml | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: sleep-multus | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: sleep-multus | |
labels: | |
app: sleep-multus | |
service: sleep-multus | |
spec: | |
ports: | |
- port: 80 | |
name: http | |
selector: | |
app: sleep-multus | |
--- | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: sleep-multus | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: sleep-multus | |
template: | |
metadata: | |
labels: | |
app: sleep-multus | |
annotations: | |
k8s.v1.cni.cncf.io/networks: '[ | |
{ "name": "macvlan-conf", | |
"ips": ["10.1.1.12/24"] } | |
]' | |
spec: | |
nodeSelector: | |
kubernetes.io/hostname: ovn-worker2 | |
terminationGracePeriodSeconds: 0 | |
serviceAccountName: sleep-multus | |
containers: | |
- name: sleep-multus | |
image: curlimages/curl | |
command: ["/bin/sleep", "infinity"] | |
imagePullPolicy: IfNotPresent | |
volumeMounts: | |
- mountPath: /etc/sleep/tls | |
name: secret-volume | |
volumes: | |
- name: secret-volume | |
secret: | |
secretName: sleep-multus-secret | |
optional: true | |
EOF | |
$ kubectl apply -f sleep-multus-default.yaml | |
# sleep | |
# not a part of the multus macvlan-conf network | |
$ cat <<EOF > sleep-nomultus-default.yaml | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: sleep | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: sleep | |
labels: | |
app: sleep | |
service: sleep | |
spec: | |
ports: | |
- port: 80 | |
name: http | |
selector: | |
app: sleep | |
--- | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: sleep | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: sleep | |
template: | |
metadata: | |
labels: | |
app: sleep | |
spec: | |
terminationGracePeriodSeconds: 0 | |
serviceAccountName: sleep | |
containers: | |
- name: sleep | |
image: curlimages/curl | |
command: ["/bin/sleep", "infinity"] | |
imagePullPolicy: IfNotPresent | |
volumeMounts: | |
- mountPath: /etc/sleep/tls | |
name: secret-volume | |
volumes: | |
- name: secret-volume | |
secret: | |
secretName: sleep-secret | |
optional: true | |
EOF | |
$ kubectl apply -f sleep-nomultus-default.yaml | |
$ kubectl create ns nosidecar | |
# sleep in nosidecar namespace on macvlan-conf multus network | |
$ cat <<EOF > sleep-httpbin-nomultus-nosidecar.yaml | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: sleep | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: sleep | |
labels: | |
app: sleep | |
service: sleep | |
spec: | |
ports: | |
- port: 80 | |
name: http | |
selector: | |
app: sleep | |
--- | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: sleep | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: sleep | |
template: | |
metadata: | |
labels: | |
app: sleep | |
annotations: | |
k8s.v1.cni.cncf.io/networks: '[ | |
{ "name": "macvlan-conf", | |
"namespace": "default", | |
"ips": ["10.1.1.13/24"] } | |
]' | |
spec: | |
nodeSelector: | |
kubernetes.io/hostname: ovn-worker2 | |
terminationGracePeriodSeconds: 0 | |
serviceAccountName: sleep | |
containers: | |
- name: sleep | |
image: curlimages/curl | |
command: ["/bin/sleep", "infinity"] | |
imagePullPolicy: IfNotPresent | |
volumeMounts: | |
- mountPath: /etc/sleep/tls | |
name: secret-volume | |
volumes: | |
- name: secret-volume | |
secret: | |
secretName: sleep-secret | |
optional: true | |
--- | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: httpbin | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: httpbin | |
labels: | |
app: httpbin | |
service: httpbin | |
spec: | |
ports: | |
- name: http | |
port: 8000 | |
targetPort: 80 | |
selector: | |
app: httpbin | |
--- | |
apiVersion: apps/v1 | |
kind: Deployment | |
metadata: | |
name: httpbin | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
app: httpbin | |
version: v1 | |
template: | |
metadata: | |
annotations: | |
k8s.v1.cni.cncf.io/networks: '[ | |
{ "name": "macvlan-conf", | |
"namespace": "default", | |
"ips": ["10.1.1.14/24"] } | |
]' | |
labels: | |
app: httpbin | |
version: v1 | |
spec: | |
nodeSelector: | |
kubernetes.io/hostname: ovn-worker | |
serviceAccountName: httpbin | |
containers: | |
- image: docker.io/kennethreitz/httpbin | |
imagePullPolicy: IfNotPresent | |
name: httpbin | |
ports: | |
- containerPort: 80 | |
EOF | |
$ kubectl apply -f sleep-httpbin-nomultus-nosidecar.yaml -n nosidecar | |
$ kubectl get svc -A | |
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE | |
default httpbin ClusterIP 10.96.92.191 <none> 8000/TCP 32m | |
default kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 49m | |
default sleep ClusterIP 10.96.164.183 <none> 80/TCP 29m | |
default sleep-multus ClusterIP 10.96.63.157 <none> 80/TCP 30m | |
istio-system istiod ClusterIP 10.96.136.7 <none> 15010/TCP,15012/TCP,443/TCP,15014/TCP 38m | |
kube-system kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 49m | |
nosidecar httpbin ClusterIP 10.96.71.12 <none> 8000/TCP 27m | |
nosidecar sleep ClusterIP 10.96.168.196 <none> 80/TCP 27m | |
ovn-kubernetes ovnkube-db ClusterIP None <none> 6641/TCP,6642/TCP 47m | |
# multus network topology | |
# default namespace | IP address | node | |
# httpbin | 10.1.1.11 | ovn-worker | |
# sleep-multus | 10.1.1.12 | ovn-worker2 | |
# sleep | 10.1.1.13 | ovn-worker2 | |
# nosidecar namespace | |
# httpbin | 10.1.1.14 | ovn-worker | |
$ export SLEEP_MULTUS_DEFAULT_POD=$(kubectl get pod -l app=sleep-multus -o jsonpath={.items..metadata.name}) | |
$ export SLEEP_DEFAULT_POD=$(kubectl get pod -l app=sleep -o jsonpath={.items..metadata.name}) | |
$ export SLEEP_NOSIDECAR_POD=$(kubectl get pod -n nosidecar -l app=sleep -o jsonpath={.items..metadata.name}) | |
$ kubectl exec -it $SLEEP_MULTUS_DEFAULT_POD -- curl http://10.1.1.11/get | |
{ | |
"args": {}, | |
"headers": { | |
"Accept": "*/*", | |
"Host": "10.1.1.11", | |
"User-Agent": "curl/8.1.1-DEV", | |
"X-B3-Parentspanid": "584b291281342c63", | |
"X-B3-Sampled": "0", | |
"X-B3-Spanid": "8919de5bdc0fc795", | |
"X-B3-Traceid": "176d409680c7ba84584b291281342c63", | |
"X-Envoy-Attempt-Count": "1" | |
}, | |
"origin": "127.0.0.6", | |
"url": "http://10.1.1.11/get" | |
} | |
$ kubectl exec -it $SLEEP_DEFAULT_POD -- curl http://10.1.1.11/get | |
This should hang. | |
kubectl exec -it $SLEEP_NOSIDECAR_POD -n nosidecar -- curl http://10.1.1.11/get 1 ↵ | |
{ | |
"args": {}, | |
"headers": { | |
"Accept": "*/*", | |
"Host": "10.1.1.11", | |
"User-Agent": "curl/8.1.1-DEV", | |
"X-B3-Sampled": "0", | |
"X-B3-Spanid": "4d0eb020ac5c6ce4", | |
"X-B3-Traceid": "a07130fe440fefd74d0eb020ac5c6ce4" | |
}, | |
"origin": "127.0.0.6", | |
"url": "http://10.1.1.11/get" | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment