tracing kubenetes data path in iptable chains

gistfile1.txt

# To create pods and policies

LOAD_COUNT=1

for cnt in $(seq 1 $LOAD_COUNT)
do
kubectl create ns policy-demo${cnt}
kubectl create deployment --namespace=policy-demo${cnt} nginx --image=nginx
kubectl scale deployment --namespace=policy-demo${cnt} nginx --replicas=2
kubectl expose --namespace=policy-demo${cnt} deployment nginx --port=80
done


# Set up testing pod (1 each)
kubectl create namespace client-allow
kubectl label namespace client-allow purpose=test
kubectl create namespace client-deny
kubectl label namespace client-deny purpose=justforfun

kubectl run --namespace=client-allow --generator=run-pod/v1 --image=busybox busybox -it  -- sh
wget -q --timeout=5 nginx.<namespace> -O -
kubectl attach busybox -n client-allow -c busybox -it

kubectl run --namespace=client-deny --generator=run-pod/v1 --image=busybox busybox -it  -- sh
wget -q --timeout=5 nginx.<namespace> -O -
kubectl attach busybox -n client-deny -c busybox -it

# First enable tracing for all traffic going out of busybox pod


# Verify on both nodes
sudo iptables-save | grep TRACE 

cd /var/log
tail -f kern.log

# Friendly print
tail -500 kern.log |  grep ID=<connID>  | tr -s ' ' | cut -d ' ' -f8,9,10,12,13
OR,
tail -500 kern.log |   awk '/<connID>/ {printf "%-40s %20s %20s %20s %20s\n", $8,$9,$10,$12,$13}'

# to inspect specific chain
sudo iptables -L <chain> -v --line-numbers


#### Network Policies. client-allow namespace can speak to nginx, but client-deny namespace cannot.
for cnt in $(seq 1 $LOAD_COUNT)
do
cat allow-busybox.yaml | sed "s/namespace: policy-demo/namespace: policy-demo${cnt}/" | kubectl apply -f -
done

# OR, Calico global network policy (gnp)
kubectl apply -f allow-busybox-gnp.yaml



#### DELETE ALL
for cnt in $(seq 1 $LOAD_COUNT)
do
kubectl delete ns policy-demo${cnt}
done

kubectl delete namespace client-allow
kubectl delete namespace client-deny

sudo iptables  -t raw -D  PREROUTING -p tcp -m tcp --dport 80 -j TRACE