deploy.sh

#!/usr/bin/env bash
set -euo pipefail

OUTPUT=deploy.yaml

# Deploy cert-manager (not requried in final version because certificates
# should be bootstrapped by the installer)
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml
kubectl wait --for condition=ready \
    -n cert-manager pod -l app.kubernetes.io/instance=cert-manager

# Namespace
cat <<EOF | tee $OUTPUT | kubectl apply -f -
---
apiVersion: v1
kind: Namespace
metadata:
  name: crio-metrics
EOF

# Permission handling (should be integrate in OpenShift later on)
oc adm policy add-scc-to-user hostnetwork -z crio-metrics -n crio-metrics

# User monitoring (should be integrated in system monitoring later on)
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: cluster-monitoring-config
  namespace: openshift-monitoring
data:
  config.yaml: |
    enableUserWorkload: true
EOF

# Certificate (should be bootstrapped by installer later on)
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: selfsigned-issuer
  namespace: crio-metrics
spec:
  selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: metrics
  namespace: crio-metrics
spec:
  subject:
    organizations:
      - crio-metrics
  dnsNames:
    - metrics
    - metrics.crio-metrics
    - metrics.crio-metrics.svc
    - metrics.crio-metrics.svc.cluster.local
  issuerRef:
    kind: Issuer
    name: selfsigned-issuer
  secretName: tls
EOF

# RBAC (for kube-rbac-proxy server)
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: crio-metrics
  namespace: crio-metrics
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: crio-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: crio-metrics
subjects:
  - kind: ServiceAccount
    name: crio-metrics
    namespace: crio-metrics
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: crio-metrics
rules:
  - apiGroups:
      - authentication.k8s.io
    resources:
      - tokenreviews
    verbs:
      - create
  - apiGroups:
      - authorization.k8s.io
    resources:
      - subjectaccessreviews
    verbs:
      - create
EOF

# RBAC (for metrics client)
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: crio-metrics-client
rules:
  - nonResourceURLs:
      - /metrics
    verbs:
      - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: crio-metrics-client
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: crio-metrics-client
subjects:
  - kind: ServiceAccount
    name: default
    namespace: crio-metrics
EOF

# DaemonSet and Service for serving the metrics
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: proxy
  namespace: crio-metrics
spec:
  selector:
    matchLabels:
      name: proxy
  template:
    metadata:
      labels:
        name: proxy
    spec:
      hostNetwork: true
      serviceAccountName: crio-metrics
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
      containers:
        - name: proxy
          image: quay.io/brancz/kube-rbac-proxy:v0.9.0
          args:
            # TODO: use https if CRI-O supports it
            - --upstream=http://127.0.0.1:9537
            - --secure-listen-address=0.0.0.0:9538
            - --v=10
            - --tls-cert-file=/tls/tls.crt
            - --tls-private-key-file=/tls/tls.key
          ports:
            - name: https
              containerPort: 9538
          volumeMounts:
            - mountPath: /tls
              name: tls
      volumes:
        - name: tls
          secret:
            secretName: tls
---
apiVersion: v1
kind: Service
metadata:
  name: metrics
  namespace: crio-metrics
  labels:
    name: metrics
spec:
  ports:
    - name: https
      port: 443
      targetPort: 9538
  selector:
    name: proxy
EOF

# Secret token for the service monitor
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
  name: token
  namespace: crio-metrics
  annotations:
    kubernetes.io/service-account.name: default
EOF

# ServiceMonitor to gather the metrics
cat <<EOF | tee -a $OUTPUT | kubectl apply -f -
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: metrics
  namespace: crio-metrics
spec:
  endpoints:
    - bearerTokenSecret:
        key: token
        name: token
      interval: 10s
      path: /metrics
      port: https
      scheme: https
      tlsConfig:
        serverName: metrics
        ca:
          secret:
            key: tls.crt
            name: tls
  selector:
    matchLabels:
      name: metrics
EOF

echo Verify that everything works:
echo "> kubectl port-forward -n openshift-user-workload-monitoring pod/prometheus-user-workload-0 9090"
echo "Open: http://localhost:9090/targets"