Explains how to scale k8s deployment based on custom external metric

A k8s custom external metrics gist.md

Kubernetes External Metric

This gist contains files used to scale a kubernetes deployment based on some other arbitrary/external metric.

index.js

const express = require('express');
const bodyParser = require('body-parser');

let externalMetricValue = Number(process.env.CUSTOM_METRIC || 0);

const prometheusMetricString = `\
# HELP custom_external_metric The value of some external metric\n\
# TYPE custom_external_metric gauge\n\
custom_external_metric \
`;

const app = express();
app.use(bodyParser.json());

app.get('/', (req, res) => res.send('Hello, World!'));

app.get('/metrics', (req, res) => {
    res.send(prometheusMetricString + externalMetricValue);
});

app.get('/value', (req, res) => res.json({ value: externalMetricValue }));

app.put('/value', (req, res) => {
    if (!req.body || typeof req.body.value === 'undefined') {
        console.error('bad request, expected body and value');
        res.status(400).json({ error: 'Bad Request', reason: 'Expected request body to have value'})
    } else {
        try {
            externalMetricValue = Number(req.body.value);
            console.log('set new metric value to ' + externalMetricValue);
            res.json({ message: 'success' });
        } catch (e) {
            console.error('error parsing number from body value', e);
            res.status(400).json({ error: 'error parsing body value' });
        }
    }
});

const port = Number(process.env.PORT || 3000);
const server = app.listen(port, () => console.log('listening on port ' + port));

const shutdown = (signal) => {
    console.log('received signal ' + signal + ', closing server');
    server.close((err) => {
        if (err) {
            console.error('error shutting down server', err);
            process.exit(1);
        } else {
            console.log('successfully shut down server');
            process.exit(0);
        }
    });
}
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));

prom-adpt-values.yaml

prometheus:
  url: 'http://prom-op-prometheus-operato-prometheus.monitoring.svc'
rules:
  custom:
  - seriesQuery: 'connection_count{namespace!="",pod!=""}'
    seriesFilters: []
    resources:
      template: <<.Resource>>
    name:
      matches: ""
      as: ""
    metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
  external:
  - seriesQuery: 'custom_external_metric{namespace!="",pod!=""}'
    seriesFilters: []
    resources:
      template: <<.Resource>>
    name:
      matches: ""
      as: "custom_external_metric"
    metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)

queue-checker-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: custom-external-metrics-emitter
spec:
  selector:
    matchLabels:
      app: custom-external-metrics-emitter-app
  template:
    metadata:
      labels:
        app: custom-external-metrics-emitter-app
    spec:
      containers:
      - name: custom-external-metrics-emitter-app
        image: briankopp/k8s-custom-external-metric-emitter:0.0.2
        resources:
          limits:
            memory: "128Mi"
            cpu: "100m"
        ports:
        - containerPort: 3000

queue-checker-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: custom-external-metrics-emitter-svc
  labels:
    hasCustomMetrics: 'true'
spec:
  selector:
    app: custom-external-metrics-emitter-app
  ports:
  - port: 3000
  type: NodePort

worker-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: object-metric-scaled-worker
spec:
  selector:
    matchLabels:
      app: object-metric-scaled-worker-app
  template:
    metadata:
      labels:
        app: object-metric-scaled-worker-app
    spec:
      containers:
      - name: object-metric-scaled-worker-app
        image: briankopp/k8s-object-metric-scaled-worker:0.0.1
        resources:
          limits:
            memory: "128Mi"
            cpu: "100m"
        ports:
        - containerPort: 3000

worker-fake.js

const delay = Number(process.env.DELAY || 5);

const runForever = () => {
    console.log('Hey, I\'m running!');
    setTimeout(runForever, delay * 1000);
};

runForever();

worker-hpa.yaml

apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: object-metric-autoscaler
spec:
  scaleTargetRef:
    # Point at the connections metric app
    apiVersion: apps/v1
    kind: Deployment
    name: object-metric-scaled-worker
  minReplicas: 1
  maxReplicas: 3
  metrics:
  - type: External
    external:
      metricName: custom_external_metric
      targetAverageValue: 500

worker-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: object-metric-scaled-worker-svc
spec:
  selector:
    app: object-metric-scaled-worker-app
  ports:
  - port: 3000
  type: NodePort