vcluster-demo

create-vcluster-for-istio.md

prerequisites

have a Rancher project with 2 namespaces where the Pods and Services seen each other

create vcluster

service exposes via IngressRoutesTCP

helm -n vc1 upgrade vc1 --set isolation.networkPolicy.enabled=false --version 0.15.7 oci://mtr.devops.telekom.de/caas/charts/vcluster
helm -n vc2 upgrade vc2 --set isolation.networkPolicy.enabled=false --version 0.15.7 oci://mtr.devops.telekom.de/caas/charts/vcluster

otherwise define Ingress wth ssl passthrough annotation

helm -n vcluster upgrade vc --set isolation.networkPolicy.enabled=false --set ingress.enabled=true --set ingress.host=vc1.otc.mcsps.de --set ingress.annotation="nginx.ingress.kubernetes.io/ssl-passthrough=true"--version 0.15.7  oci://mtr.devops.telekom.de/caas/charts/vcluster

ingress-vc1.yaml

apiVersion: traefik.containo.us/v1alpha1
kind: IngressRouteTCP
metadata:
  creationTimestamp: "2023-12-27T20:38:15Z"
  generation: 6
  name: vc1
  namespace: vc1
  resourceVersion: "18925647"
  uid: 9d51bb93-6699-4816-8545-53321cb93e11
spec:
  entryPoints:
  - websecure
  routes:
  - match: HostSNI(`vc1.otc.mcsps.de`)
    services:
    - name: vc1
      port: 443
  tls:
    passthrough: true

kubectl apply -f ingress-vc1.yaml

ref: traefik/traefik#7112

docs istio

• https://istio.io/latest/docs/setup/install/multicluster/before-you-begin/
• https://aungzanbaw.medium.com/a-step-by-step-guide-to-creating-users-in-kubernetes-6a5a2cfd8c71
• https://istio.io/latest/docs/tasks/security/cert-management/plugin-ca-cert/

enable trust

make -f ../istio//tools/certs/Makefile.selfsigned.mk root-ca
make -f ../istio//tools/certs/Makefile.selfsigned.mk vc1-cacerts
make -f ../istio//tools/certs/Makefile.selfsigned.mk vc2-cacerts

vcluster -n vc1 connect vc1 -- bash
source  <(kubectl completion bash)

kubectl -n istio-system create secret generic cacerts --from-file=vc1/ca-cert.pem --from-file=vc1/ca-key.pem --from-file=vc1/root-cert.pem --from-file=vc1/cert-chain.pem

vc-rbac.yaml

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: vc-istio-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: admin
subjects:
- name: u-istio
  kind: User
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f vc-rbac.yaml

openssl genpkey -out u-istio.key -algorithm Ed25519
openssl req -new -key u-istio.key -out u-istio.csr -subj "/CN=u-istio/O=admin"
cat u-istio.csr | base64 | tr -d "\n"

signrequest.yaml

apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: u-istio
spec:
  request: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlHaU1GWUNBUUF3SXpFUk1BOEdBMVVFQXd3SWRTMXBjM1JwYnl3eERqQU1CZ05WQkFvTUJXRmtiV2x1TUNvdwpCUVlESzJWd0F5RUFXeEZ3QklOVnIyUklPTGxHS0d4K3JVLzdQV1EzcWhpVUNJTmZNM1paWXVpZ0FEQUZCZ01yClpYQURRUUFhKzZPMFRUdXZ2TzF1dzhaQjdhekZYcFJRRVBpblJXYkFSSXFPbHY3WDZidWZqK3hJSlpieWtOZUYKYXNMNU1ZY1RXdVhSQVJJRU16Zi9hSGhDTGdzRAotLS0tLUVORCBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K
  signerName: kubernetes.io/kube-apiserver-client
  expirationSeconds: 31536000  # 365 day
  usages:
  - client auth

kubectl apply -f signrequest.yaml
kubectl certificate approve u-istio

kubectl get csr/u-istio -o jsonpath="{.status.certificate}"

cat u-istio.key | base64 -w 0

kubectl -n vcluster exec -it vc-0 -- cat  /data/server/tls/server-ca.crt| base64 -w 0

KUBECONFIG

apiVersion: v1
clusters:
- name: "vc0"
  cluster:
    server: "https://vc1.otc.mcsps.de:443"
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTURJd01UYzROamN3SGhjTk1qTXhNakE0TURZME5ESTNXaGNOTXpNeE1qQTFNRFkwTkRJMwpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTURJd01UYzROamN3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUR3JDdi8vSktUekh3eklOS0p3S2QyVE9zSkhTT080bmRHb0tOVDI4YlQKYW03c2hQTFhpYy9wVmVnc3dlekhRWnpNVWZJMVhDMkJqVXAxeis3V0h6eEhvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVFaR0NrWXF4VGV4YUp6Tm5uQTZtCngzUEVkQll3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUpobC9VNDlmMEtrcFdLNnNsd1dWLy8wWEVITEFtUzEKYnF1TnBjSDFMVEpxQWlCcmxqbFpNNVNmNjU0TTNTb2lwOU5tV1R0emJXTDQrUEVUTy9WYVJIMVphQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
contexts:
- name: "vc0"
  context:
    user: "u-istio"
    cluster: "vc0"
current-context: "vc0"
kind: Config
preferences: {}
users:
- name: u-istio
  user:
    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJXekNDQVFHZ0F3SUJBZ0lSQUpvbjJFT2haY0JON2EvN2hRRGlUWU13Q2dZSUtvWkl6ajBFQXdJd0l6RWgKTUI4R0ExVUVBd3dZYXpOekxXTnNhV1Z1ZEMxallVQXhOekF5TURFM09EWTNNQjRYRFRJek1USXlOekl5TVRBMQpNbG9YRFRJME1USXlOakl5TVRBMU1sb3dJakVPTUF3R0ExVUVDaE1GWVdSdGFXNHhFREFPQmdOVkJBTVRCM1V0CmFYTjBhVzh3S2pBRkJnTXJaWEFESVFCYkVYQUVnMVd2WkVnNHVVWW9iSDZ0VC9zOVpEZXFHSlFJZzE4emRsbGkKNktOR01FUXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWZCZ05WSFNNRQpHREFXZ0JSS0JjeUNPQjNNNTlaNHUxNjVsbDRKa1ZPaHBUQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUErVmlXClovRE9SZnpNZVNzUzk3TlpZNFgzL3ZhNFQwSFFlTTZaeldBTjh1OENJRHVnaURBbDNKbjJIbGJPSDNXa1IySlgKTXhhS1orVzRkL1FmQytOb1J1OWkKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
    client-key-data: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUNzQnJ1c3hTcmdyWFB3b0dlVzdJUTNmZDlhRTZ0RlFxa3hVZTlIVmFiUWoKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=

complete for second cluster:

apiVersion: v1
clusters:
- cluster:
   certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTURJd01UYzROamN3SGhjTk1qTXhNakE0TURZME5ESTNXaGNOTXpNeE1qQTFNRFkwTkRJMwpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTURJd01UYzROamN3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUR3JDdi8vSktUekh3eklOS0p3S2QyVE9zSkhTT080bmRHb0tOVDI4YlQKYW03c2hQTFhpYy9wVmVnc3dlekhRWnpNVWZJMVhDMkJqVXAxeis3V0h6eEhvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVFaR0NrWXF4VGV4YUp6Tm5uQTZtCngzUEVkQll3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUpobC9VNDlmMEtrcFdLNnNsd1dWLy8wWEVITEFtUzEKYnF1TnBjSDFMVEpxQWlCcmxqbFpNNVNmNjU0TTNTb2lwOU5tV1R0emJXTDQrUEVUTy9WYVJIMVphQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
   server: https://vc1.otc.mcsps.de:443
 name: vc0
- cluster:
   certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlCZGpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5DQpkbVZ5TFdOaFFERTNNRE14TnpRNU56RXdIaGNOTWpNeE1qSXhNVFl3T1RNeFdoY05Nek14TWpFNE1UWXdPVE14DQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTURNeE56UTVOekV3V1RBVEJnY3Foa2pPDQpQUUlCQmdncWhrak9QUU1CQndOQ0FBUUE2ZjdMZzhmdGtuTmNEU25jS01EMjA1dDBaekp6UU9Eb1NNREpjajBjDQpGcUlrWkdvZ0tmdklvNVZsdHliRFhyU3lRZll1REx3Q2J2am1NdTNGTEpOQm8wSXdRREFPQmdOVkhROEJBZjhFDQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVTNtTVM4bnhPaC9IbldKZlV0Sm5WDQpBbVN5SUdrd0NnWUlLb1pJemowRUF3SURSd0F3UkFJZ2VXcUpPTDhYUkxXZ2Q4dkp6cm5sdXBoVmpVK1BPTEVrDQpMdDVoVWVEMGgrTUNJQXZLckdGbGFlOXIxNVBrK2QwT2Izazh0SVhiRHZEd1VRRG8zSUtEZTZ1OA0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ0K
   server: https://vc2.otc.mcsps.de:443
 name: vc2
contexts:
- context:
   cluster: vc0
   user: u-istio
 name: vc0
- context:
   cluster: vc2
   user: u-istio-2
 name: vc2
current-context: vc2
kind: Config
preferences: {}
users:
- name: u-istio
 user:
   client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJXekNDQVFHZ0F3SUJBZ0lSQUpvbjJFT2haY0JON2EvN2hRRGlUWU13Q2dZSUtvWkl6ajBFQXdJd0l6RWgKTUI4R0ExVUVBd3dZYXpOekxXTnNhV1Z1ZEMxallVQXhOekF5TURFM09EWTNNQjRYRFRJek1USXlOekl5TVRBMQpNbG9YRFRJME1USXlOakl5TVRBMU1sb3dJakVPTUF3R0ExVUVDaE1GWVdSdGFXNHhFREFPQmdOVkJBTVRCM1V0CmFYTjBhVzh3S2pBRkJnTXJaWEFESVFCYkVYQUVnMVd2WkVnNHVVWW9iSDZ0VC9zOVpEZXFHSlFJZzE4emRsbGkKNktOR01FUXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWZCZ05WSFNNRQpHREFXZ0JSS0JjeUNPQjNNNTlaNHUxNjVsbDRKa1ZPaHBUQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUErVmlXClovRE9SZnpNZVNzUzk3TlpZNFgzL3ZhNFQwSFFlTTZaeldBTjh1OENJRHVnaURBbDNKbjJIbGJPSDNXa1IySlgKTXhhS1orVzRkL1FmQytOb1J1OWkKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
   client-key-data: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUNzQnJ1c3hTcmdyWFB3b0dlVzdJUTNmZDlhRTZ0RlFxa3hVZTlIVmFiUWoKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=
- name: u-istio-2
 user:
   client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJXVENDQVFDZ0F3SUJBZ0lRV0xkKzZGaG1wbTF3SW1BQmplb0QzakFLQmdncWhrak9QUVFEQWpBak1TRXcKSHdZRFZRUUREQmhyTTNNdFkyeHBaVzUwTFdOaFFERTNNRE14TnpRNU56RXdIaGNOTWpNeE1qSTNNakl6TURFMApXaGNOTWpReE1qSTJNakl6TURFMFdqQWlNUTR3REFZRFZRUUtFd1ZoWkcxcGJqRVFNQTRHQTFVRUF4TUhkUzFwCmMzUnBiekFxTUFVR0F5dGxjQU1oQUZzUmNBU0RWYTlrU0RpNVJpaHNmcTFQK3oxa042b1lsQWlEWHpOMldXTG8KbzBZd1JEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFNQmdOVkhSTUJBZjhFQWpBQU1COEdBMVVkSXdRWQpNQmFBRkFnbEdFSDFkcDVZZVYrRkZJVGF5RkhoQkZQT01Bb0dDQ3FHU000OUJBTUNBMGNBTUVRQ0lHc1lKRXZoCmxzN0daZVBNajc4Z2lSckRTSThoZ3FmZTNDdHpRbU80OHR0eEFpQk5HNmNpS201anhMcmtuUWNlSFkybDRpMVgKZHNSZ1RTZVNOS2Jia2FHVUpRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
   client-key-data: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUNzQnJ1c3hTcmdyWFB3b0dlVzdJUTNmZDlhRTZ0RlFxa3hVZTlIVmFiUWoKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=

istioctl create-remote-secret --context=vc0 --name=vc0 | kubectl apply -f - --context=vc2
istioctl create-remote-secret --context=vc2 --name=vc2 | kubectl apply -f - --context=vc0   

install istio without operator

istiooperator.yaml

apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
metadata:
  annotations:
    install.istio.io/ignoreReconcile: "false"
  name: istiooperator
  namespace: istio-system
spec:
  components:
    base:
      enabled: true
    cni:
      enabled: false
    egressGateways:
    - enabled: false
      name: istio-egressgateway
    ingressGateways:
    - enabled: true
      name: istio-ingressgateway
    istiodRemote:
      enabled: false
    pilot:
      enabled: true
  hub: docker.io/istio
  meshConfig:
    defaultConfig:
      proxyMetadata: {}
    enablePrometheusMerge: true
  profile: minimal
  tag: 1.20.1
  values:
    base:
      enableCRDTemplates: false
      validationURL: ""
    defaultRevision: ""
    gateways:
      istio-egressgateway:
        autoscaleEnabled: false
        env: {}
        name: istio-egressgateway
        secretVolumes:
        - mountPath: /etc/istio/egressgateway-certs
          name: egressgateway-certs
          secretName: istio-egressgateway-certs
        - mountPath: /etc/istio/egressgateway-ca-certs
          name: egressgateway-ca-certs
          secretName: istio-egressgateway-ca-certs
        type: ClusterIP
      istio-ingressgateway:
        autoscaleEnabled: false
        env: {}
        name: istio-ingressgateway
        secretVolumes:
        - mountPath: /etc/istio/ingressgateway-certs
          name: ingressgateway-certs
          secretName: istio-ingressgateway-certs
        - mountPath: /etc/istio/ingressgateway-ca-certs
          name: ingressgateway-ca-certs
          secretName: istio-ingressgateway-ca-certs
        type: ClusterIP
    global:
      configValidation: true
      defaultNodeSelector: {}
      defaultPodDisruptionBudget:
        enabled: true
      defaultResources:
        requests:
          cpu: 10m
      imagePullPolicy: ""
      imagePullSecrets: []
      istioNamespace: istio-system
      istiod:
        enableAnalysis: false
      jwtPolicy: third-party-jwt
      logAsJson: false
      logging:
        level: default:error
      meshID: mesh1
      meshNetworks: {}
      mountMtlsCerts: false
      multiCluster:
        clusterName: vc0
        enabled: true
      network: network1
      omitSidecarInjectorConfigMap: false
      oneNamespace: false
      operatorManageWebhooks: false
      pilotCertProvider: istiod
      proxy:
        autoInject: enabled
        clusterDomain: cluster.local
        componentLogLevel: misc:error
        enableCoreDump: false
        excludeIPRanges: ""
        excludeInboundPorts: ""
        excludeOutboundPorts: ""
        image: proxyv2
        includeIPRanges: '*'
        logLevel: warning
        privileged: false
        readinessFailureThreshold: 4
        readinessInitialDelaySeconds: 0
        readinessPeriodSeconds: 15
        resources:
          limits:
            cpu: 2000m
            memory: 1024Mi
          requests:
            cpu: 100m
            memory: 128Mi
        startupProbe:
          enabled: true
          failureThreshold: 600
        statusPort: 15020
        tracer: zipkin
      proxy_init:
        image: proxyv2
      useMCP: false
    pilot:
      autoscaleEnabled: false
      image: pilot
    telemetry:
      enabled: false

kubectl -n istio-system apply -f istiooperator.yaml

check logs

kubectl -n istio-system logs -l app=istiod -f
kubectl -n istio-system logs -l app=istio-ingressgateway -f

install the demo app helloworöd v1/v2

https://istio.io/latest/docs/setup/install/multicluster/verify/

verify cross-cluster traffic

$ for i in {1..12}; do kubectl -n sample exec -it sleep-76cc9846f7-vtm4r -- curl -sS h
elloworld.sample:5000/hello;done
Hello version: v1, instance: helloworld-v1-54864596f9-7897x
Hello version: v1, instance: helloworld-v1-54864596f9-7897x
Hello version: v2, instance: helloworld-v2-c4b799cd-4zq7h
Hello version: v2, instance: helloworld-v2-c4b799cd-4zq7h
Hello version: v1, instance: helloworld-v1-54864596f9-7897x
Hello version: v1, instance: helloworld-v1-54864596f9-7897x
Hello version: v2, instance: helloworld-v2-c4b799cd-4zq7h
Hello version: v1, instance: helloworld-v1-54864596f9-7897x
Hello version: v2, instance: helloworld-v2-c4b799cd-4zq7h
Hello version: v1, instance: helloworld-v1-54864596f9-7897x
Hello version: v2, instance: helloworld-v2-c4b799cd-4zq7h
Hello version: v2, instance: helloworld-v2-c4b799cd-4zq7h

dashboard.yaml

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: Namespace
metadata:
  name: kubernetes-dashboard

---

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kubernetes-dashboard
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kubernetes-dashboard
type: Opaque
data:
  csrf: ""

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-key-holder
  namespace: kubernetes-dashboard
type: Opaque

---

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-settings
  namespace: kubernetes-dashboard

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
rules:
  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
    verbs: ["get", "update", "delete"]
    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["kubernetes-dashboard-settings"]
    verbs: ["get", "update"]
    # Allow Dashboard to get metrics.
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["heapster", "dashboard-metrics-scraper"]
    verbs: ["proxy"]
  - apiGroups: [""]
    resources: ["services/proxy"]
    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
    verbs: ["get"]

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
rules:
  # Allow Metrics Scraper to get metrics from the Metrics server
  - apiGroups: ["metrics.k8s.io"]
    resources: ["pods", "nodes"]
    verbs: ["get", "list", "watch"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      securityContext:
        fsGroup: 2001
        supplementalGroups:
        - 2001
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: kubernetes-dashboard
          image: mtr.devops.telekom.de/caas/kubernetes-dashboard:v2.7.0
          imagePullPolicy: Always
          ports:
            - containerPort: 8443
              protocol: TCP
          args:
            - --auto-generate-certificates
            - --namespace=kubernetes-dashboard
            # Uncomment the following line to manually specify Kubernetes API server Host
            # If not specified, Dashboard will attempt to auto discover the API server and connect
            # to it. Uncomment only if the default does not work.
            # - --apiserver-host=http://my-address:port
          volumeMounts:
            - name: kubernetes-dashboard-certs
              mountPath: /certs
              # Create on-disk volume to store exec logs
            - mountPath: /tmp
              name: tmp-volume
          livenessProbe:
            httpGet:
              scheme: HTTPS
              path: /
              port: 8443
            initialDelaySeconds: 30
            timeoutSeconds: 30
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      volumes:
        - name: kubernetes-dashboard-certs
          secret:
            secretName: kubernetes-dashboard-certs
        - name: tmp-volume
          emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 8000
      targetPort: 8000
  selector:
    k8s-app: dashboard-metrics-scraper

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: dashboard-metrics-scraper
  template:
    metadata:
      labels:
        k8s-app: dashboard-metrics-scraper
    spec:
      securityContext:
        fsGroup: 2001
        supplementalGroups:
        - 2001
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: dashboard-metrics-scraper
          image: mtr.devops.telekom.de/caas/kubernetes-dashboard-metrics:v1.0.8
          ports:
            - containerPort: 8000
              protocol: TCP
          livenessProbe:
            httpGet:
              scheme: HTTP
              path: /
              port: 8000
            initialDelaySeconds: 30
            timeoutSeconds: 30
          volumeMounts:
          - mountPath: /tmp
            name: tmp-volume
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      volumes:
        - name: tmp-volume
          emptyDir: {}
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
---
# kubectl -n kubernetes-dashboard create token admin-user

ingressroute-dashboard.yaml

apiVersion: traefik.containo.us/v1alpha1
kind: IngressRouteTCP
metadata:
  name: vc2dashboard
  namespace: vcluster2
spec:
  entryPoints:
  - websecure
  routes:
  - match: HostSNI(`vc2-dashboard.otc.mcsps.de`)
    services:
    - name: kubernetes-dashboard-x-kubernetes-dashboard-x-vc2
      port: 443
  tls:
    passthrough: true