Kubernetes Connection Count Custom Metrics

A k8s custom metrics gist.md

K8s Custom Metrics

These files accompany a set of blog posts describing kubernetes custom metrics.

deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: connection-metrics-app
spec:
  selector:
    matchLabels:
      app: connection-metrics-server
  template:
    metadata:
      labels:
        app: connection-metrics-server
    spec:
      containers:
      - name: connection-metrics-server
        image: briankopp/k8s-custom-connections-metric:1.0.0
        resources:
          limits:
            memory: "128Mi"
            cpu: "100m"
        ports:
        - containerPort: 3000

Dockerfile

FROM node:10-alpine

RUN mkdir /usr/src && \
    mkdir /usr/src/node_modules && \
    chown node:node -R /usr/src

WORKDIR /usr/src
USER node

COPY package*.json ./
RUN npm install

COPY . ./
EXPOSE 3000
ENTRYPOINT ["node", "index.js"]

horizontal-pod-autoscaler.yaml

apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: connections-metric-hpa
spec:
  scaleTargetRef:
    # Point at the connections metric app
    apiVersion: apps/v1
    kind: Deployment
    name: connection-metrics-app
  minReplicas: 1
  maxReplicas: 20
  metrics:
  - type: Pods # use Pods metric, since the metric is attached to the pod
    pods:
      targetAverageValue: 5
      metricName: connection_count

index.js

const CONNECTION_LIMIT = Number(process.env.CONNECTION_LIMIT || 20);
const STICKY_TTL = Number(process.env.STICKY_TTL || 5);
const express = require('express');

let connectionCount = 0;
const prometheusMetricString = `\
# HELP cm_connection_count The value of some external metric\n\
# TYPE cm_connection_count gauge\n\
connection_count \
`;

const app = express();

app.get('/', (req, res) => res.send('Hello, World!'));
app.get('/connection', (req, res) => {
    console.log('Received connection request');
    connectionCount++;
    let responseMsg = 'Here\'s a new connection! I have ' + connectionCount + ' connections so far!';
    if (connectionCount > CONNECTION_LIMIT) {
        // simulate disconnecting the connection
        console.log('More connections than desired, schedule reduction');
        setTimeout(() => {
            console.log('Reducing connection');
            connectionCount--;
        }, STICKY_TTL * 2 * 1000);
        responseMsg += '\nI have more connections than I want, reduction scheduled.';
    }
    res.send(responseMsg);
});

app.get('/metrics', (req, res) => {
    res.send(prometheusMetricString + connectionCount);
});

const port = Number(process.env.PORT || 3000);
const server = app.listen(port, () => console.log('listening on port ' + port));

const shutdown = (signal) => {
    console.log('received signal ' + signal + ', closing server');
    server.close((err) => {
        if (err) {
            console.error('error shutting down server', err);
            process.exit(1);
        } else {
            console.log('successfully shut down server');
            process.exit(0);
        }
    });
}
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));

prom-adpt-values.yaml

prometheus:
  url: 'http://prom-op-prometheus-operato-prometheus.monitoring.svc'
rules:
  custom:
  - seriesQuery: 'connection_count{namespace!="",pod!=""}'
    seriesFilters: []
    resources:
      template: <<.Resource>>
    name:
      matches: ""
      as: ""
    metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)

service-monitor.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: node-apps-service-monitor
  namespace: monitoring
  labels:
    release: prom-op
spec:
  endpoints:
  - targetPort: 3000
    honorLabels: true
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      hasCustomMetrics: 'true'

service.yaml

apiVersion: v1
kind: Service
metadata:
  name: connections-metric-svc
  labels:
    hasCustomMetrics: 'true'
spec:
  selector:
    app: connection-metrics-server
  ports:
  - port: 3000
  type: NodePort