Multi-tenant Kubernetes Clusters with the HAProxy Kubernetes Ingress Controller

20200312-01.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: dev

20200312-02.sh

$ kubectl apply -f dev-namespace.yaml

20200312-03.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: my-configmap
  namespace: dev
data:
  foo: 'bar'

20200312-04.sh

$ kubectl get configmaps --namespace=dev

20200312-05.sh

$ kubectl get namespaces
NAME     STATUS   AGE
default  Active   4m33s
dev      Active   2m38s

20200312-06.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: dev-edit
  namespace: dev # grants permissions within the "dev" namespace
subjects:
- kind: User
  name: bob # permissions for a user named bob
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: edit # read/write access
  apiGroup: rbac.authorization.k8s.io

20200312-07.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: dev-edit
  namespace: dev # grants permissions within the "dev" namespace
subjects:
- kind: Group
  name: dev-group # permissions for the group
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: edit # read/write access
  apiGroup: rbac.authorization.k8s.io

20200312-08.sh

$ kubectl apply -f dev-rolebinding.yaml

20200312-09.sh

# Sign it with the cluster's CA certificate
# This creates bob.crt
$ openssl x509 -req -in bob.csr -CA ~/.minikube/ca.crt -CAkey ~/.minikube/ca.key -CAcreateserial -out bob.crt -days 1000

20200312-10.sh

$ kubectl config set-credentials bob --client-certificate=bob.crt --client-key=bob.key

$ kubectl config set-context minikube-bob --cluster=minikube --user=bob

$ kubectl config  use-context minikube-bob

20200312-11.sh

$ kubectl get pods --namespace=dev
NAME                                       READY   STATUS    RESTARTS   AGE
app-66d9457bf5-vpbnw                       1/1     Running   1          22h

$ kubectl get pods --namespace=default
Error from server (Forbidden): pods is forbidden: User "bob" cannot list resource "pods" in API group "" in the namespace "default"

20200312-12.sh

$ kubectl config  use-context minikube

20200312-13.sh

$ helm install onlydev haproxytech/kubernetes-ingress \
  --set-string "controller.extraArgs={--namespace-whitelist=dev}"

20200312-14.sh

$ helm install onlydev haproxytech/kubernetes-ingress \
  --set-string "controller.extraArgs={--namespace-whitelist=dev-team-a,--namespace-whitelist=dev-team-b}"

20200312-15.yaml

apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: app-ingress
  namespace: dev
spec:
  rules:
  - http:
      paths:
      - path: /app-service
        backend:
          serviceName: app-service
          servicePort: 80

20200312-16.sh

$ helm install intranet haproxytech/kubernetes-ingress \
  --set controller.ingressClass=intranet

20200312-17.yaml

apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: app-ingress-internal
  namespace: default
  annotations:
    haproxy.org/ingress.class: "intranet"
spec:
  rules:
  - http:
      paths:
      - path: /app-service
        backend:
          serviceName: app-service-internal
          servicePort: 80

20200312-18.yaml

apiVersion: v1
kind: ResourceQuota
metadata:
  name: dev-resources
  namespace: dev
spec:
  hard:
    requests.cpu: "1"
    requests.memory: 1Gi
    limits.cpu: "2"
    limits.memory: 2Gi

20200312-19.sh

$ kubectl apply -f dev-quota.yaml

20200312-20.sh

$ kubectl describe resourcequota dev-resources -n dev
Name:            dev-resources
Namespace:       dev
Resource         Used  Hard
--------         ----  ----
limits.cpu       0     2
limits.memory    0     2Gi
requests.cpu     500m  1
requests.memory  50Mi  1Gi

20200312-21.sh

# Create a certificate signing request with CN=bob and O=dev-group
# This creates bob.csr and bob.key
$ openssl req -newkey rsa:2048 -nodes -keyout bob.key -out bob.csr -subj "/CN=bob/O=dev-group"

20200312-22.sh

$ cat <<EOF | kubectl apply -f -
apiVersion: certificates.k8s.io/v1beta1
kind: CertificateSigningRequest
metadata:
  name: bob
spec:
  request: $(cat bob.csr | base64 | tr -d '\n')
  usages:
  - digital signature
EOF

20200312-23.sh

$ kubectl certificate approve bob

20200312-24.sh

$ kubectl get csr bob -o jsonpath='{.status.certificate}' | base64 --decode > bob.crt