vmagent k8s discovery timings

README.md

how to generate load on vmagent

requirements

• golang
• python3
• git

starting

# clone api server
git clone project https://github.com/alvaroaleman/static-kas

# build fake kubernetes api server
go build static-kas/cmd/main.go
# create dir and generate objects
mkdir -p static-kas/data/namespaces/load-test/core
# generate
python3 generate_objects.py static-kas/data/namespaces/load-test/core
# start fake api-server, it starts on 8080 port.
./static-kas/main -base-dir static-kas/data

# start vmagent, python saves scrape config into /tmp folder
./bin/vmagent --promscrape.config /tmp/vmagent_debug.yaml --promscrape.suppressDuplicateScrapeTargetErrors --promscrape.suppressScrapeErrors --remoteWrite.url http://localhost:8428/api/v1/write

for kubectl cluster access use following syntax kubectl config set-cluster static-kas --server=http://localhost:8080 --insecure-skip-tls-verify

generate_objects.py

import os.path

from jinja2 import Template
pod_template='''
apiVersion: v1
items:
{% for n in range(1, pod_count ) %}
- apiVersion: v1
  kind: Pod
  metadata:
    annotations:
      k8s.v1.cni.cncf.io/network-status: |-
        [{
            "name": "openshift-sdn",
            "interface": "eth0",
            "ips": [
                "10.133.0.2"
            ],
            "default": true,
            "dns": {}
        }]
      k8s.v1.cni.cncf.io/networks-status: |-
        [{
            "name": "openshift-sdn",
            "interface": "eth0",
            "ips": [
                "10.133.0.2"
            ],
            "default": true,
            "dns": {}
        }]
      openshift.io/scc: anyuid
    creationTimestamp: "2022-03-04T18:05:22Z"
    generateName: service-ca-operator-7496fb6588-
    labels:
      app: service-ca-operator-{{n}}
      pod-template-hash: 7496fb6588
    name: service-ca-operator-7496fb6588-2zznl-{{n}}
    namespace: for-test
    ownerReferences:
    - apiVersion: apps/v1
      blockOwnerDeletion: true
      controller: true
      kind: ReplicaSet
      name: service-ca-operator-7496fb6588
      uid: 6c7e7d19-caba-4a27-b444-3dd261291b1c
    resourceVersion: "10141"
    uid: 86219913-4236-4799-9364-524bbc9c{{n}}
  spec:
    containers:
    - args:
      - --config=/var/run/configmaps/config/operator-config.yaml
      - -v=4
      command:
      - service-ca-operator
      - operator
      env:
      - name: CONTROLLER_IMAGE
        value: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bae0d375f54ca2a0bf5fac29d9aff7a99e6b1a1c85874d51ad1142a9b31cad02
      - name: OPERATOR_IMAGE_VERSION
        value: 4.10.0-0.nightly-2022-03-04-174335
      image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bae0d375f54ca2a0bf5fac29d9aff7a99e6b1a1c85874d51ad1142a9b31cad02
      imagePullPolicy: IfNotPresent
      name: service-ca-operator
      resources:
        requests:
          cpu: 10m
          memory: 80Mi
      securityContext:
        capabilities:
          drop:
          - MKNOD
      terminationMessagePath: /dev/termination-log
      terminationMessagePolicy: File
      volumeMounts:
      - mountPath: /var/run/configmaps/config
        name: config
      - mountPath: /var/run/secrets/serving-cert
        name: serving-cert
      - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
        name: kube-api-access-c8pwl
        readOnly: true
    dnsPolicy: ClusterFirst
    enableServiceLinks: true
    imagePullSecrets:
    - name: service-ca-operator-dockercfg-9bj52
    nodeName: ip-10-0-131-24.ec2.internal
    preemptionPolicy: PreemptLowerPriority
    priority: 2000000000
    priorityClassName: system-cluster-critical
    restartPolicy: Always
    schedulerName: default-scheduler
    securityContext:
      runAsGroup: 1001
      runAsNonRoot: true
      runAsUser: 1001
      seLinuxOptions:
        level: s0:c17,c4
    serviceAccount: service-ca-operator
    serviceAccountName: service-ca-operator
    terminationGracePeriodSeconds: 30
    tolerations:
    - effect: NoSchedule
      key: node-role.kubernetes.io/master
      operator: Exists
    - effect: NoExecute
      key: node.kubernetes.io/unreachable
      operator: Exists
      tolerationSeconds: 120
    - effect: NoExecute
      key: node.kubernetes.io/not-ready
      operator: Exists
      tolerationSeconds: 120
    - effect: NoSchedule
      key: node.kubernetes.io/memory-pressure
      operator: Exists
    volumes:
    - name: serving-cert
      secret:
        defaultMode: 420
        optional: true
        secretName: serving-cert
    - configMap:
        defaultMode: 420
        name: service-ca-operator-config
      name: config
    - name: kube-api-access-c8pwl
      projected:
        defaultMode: 420
        sources:
        - serviceAccountToken:
            expirationSeconds: 3607
            path: token
        - configMap:
            items:
            - key: ca.crt
              path: ca.crt
            name: kube-root-ca.crt
        - downwardAPI:
            items:
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
              path: namespace
        - configMap:
            items:
            - key: service-ca.crt
              path: service-ca.crt
            name: openshift-service-ca.crt
  status:
    conditions:
    - lastProbeTime: null
      lastTransitionTime: "2022-03-04T18:06:54Z"
      status: "True"
      type: Initialized
    - lastProbeTime: null
      lastTransitionTime: "2022-03-04T18:08:36Z"
      status: "True"
      type: Ready
    - lastProbeTime: null
      lastTransitionTime: "2022-03-04T18:08:36Z"
      status: "True"
      type: ContainersReady
    - lastProbeTime: null
      lastTransitionTime: "2022-03-04T18:06:54Z"
      status: "True"
      type: PodScheduled
    containerStatuses:
    - containerID: cri-o://b977d706f10eb046d4bb31db011143a80d2e68c05a7da4ba698f4106675933f0
      image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bae0d375f54ca2a0bf5fac29d9aff7a99e6b1a1c85874d51ad1142a9b31cad02
      imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bae0d375f54ca2a0bf5fac29d9aff7a99e6b1a1c85874d51ad1142a9b31cad02
      lastState:
        terminated:
          containerID: cri-o://a615e352077d8ee118533c2ff23bc4b0e31135ff6a1a2270990e21cb0e9484c0
          exitCode: 1
          finishedAt: "2022-03-04T18:08:34Z"
          reason: Error
          startedAt: "2022-03-04T18:06:59Z"
      name: service-ca-operator
      ready: true
      restartCount: 1
      started: true
      state:
        running:
          startedAt: "2022-03-04T18:08:35Z"
    hostIP: 10.0.131.24
    phase: Running
    podIP: 10.133.0.{{n}}
    podIPs:
    - ip: 10.133.0.{{n}}
    qosClass: Burstable
    startTime: "2022-03-04T18:06:54Z"
{% endfor %}
kind: PodList
metadata:
  resourceVersion: "188166"
'''

#
endpoint_template='''
apiVersion: v1
items:
{% set count = namespace(value=1) %}
{% for n in range(1, ep_count ) %}
- apiVersion: v1
  kind: Endpoints
  metadata:
    annotations:
      endpoints.kubernetes.io/last-change-trigger-time: "2022-03-21T20:50:22Z"
    creationTimestamp: "2021-11-11T05:36:33Z"
    labels:
      instance-num: ber-{{n}}
      app.kubernetes.io/instance: operator
      app.kubernetes.io/managed-by: Helm
      app.kubernetes.io/name: victoria-metrics-operator
      helm.sh/chart: victoria-metrics-operator-0.4.2
    name: service-ca-operator-{{n}}
    namespace: for-test
    resourceVersion: "147460715"
    uid: 1a6a2431-00a5-48b5-b480-af17dab5a194
  subsets:
  - addresses:
  {%- for c in range(0, pod_count) %}
    - ip: 10.70.0.1{{cnt}}
      nodeName: gk3-main-autopilot-nap-1e6pqze5-624b44b8-nd94
      targetRef:
        kind: Pod
        name: service-ca-operator-7496fb6588-2zznl-{{ count.value }}
        namespace: for-test
        resourceVersion: "147460711"
        uid: 86219913-4236-4799-9364-524bbc9c{{count.value}}
    {% set count.value = count.value + 1 %}
    {%- endfor %}
    ports:
    - name: webhook
      port: 9443
      protocol: TCP
    - name: http
      port: 8080
      protocol: TCP
{% endfor %}
kind: EndpointList
metadata:
  resourceVersion: "188166"
'''
service_template='''
apiVersion: v1
items:
{% for n in range(1, service_count ) %}
- apiVersion: v1
  kind: Service
  metadata:
    annotations:
      meta.helm.sh/release-name: operator
      meta.helm.sh/release-namespace: monitoring
    creationTimestamp: "2021-11-11T05:36:33Z"
    labels:
      app.kubernetes.io/instance: operator
      app.kubernetes.io/managed-by: Helm
      app.kubernetes.io/name: victoria-metrics-operator
      helm.sh/chart: victoria-metrics-operator-0.4.2
    name: service-ca-operator-{{n}}
    namespace: for-test
    resourceVersion: "50178749"
    uid: 87bdcc33-5197-4493-a78b-82813f7b4280
  spec:
    clusterIP: 10.70.131.{{n}}
    clusterIPs:
    - 10.70.131.{{n}}
    ipFamilies:
    - IPv4
    ipFamilyPolicy: SingleStack
    ports:
    - name: http
      port: 8080
      protocol: TCP
      targetPort: 8080
    - name: webhook
      port: 443
      protocol: TCP
      targetPort: 9443
    selector:
      app.kubernetes.io/instance: operator
      app.kubernetes.io/name: victoria-metrics-operator
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
{% endfor %}
kind: ServiceList
metadata:
  resourceVersion: "188166"
'''

scrape_config='''
scrape_configs:
{% for n in range(0, service_count) %}
  - job_name: vmservicescrape/pod/{{n}}
    honor_labels: false
    kubernetes_sd_configs:
      - role: endpoints
        api_server: http://localhost:8080
    relabel_configs:
      - action: keep
        source_labels:
         - __meta_kubernetes_service_name
        regex: "service-ca-operator-{{n}}"
      - source_labels:
          - __meta_kubernetes_namespace
        target_label: namespace
      - source_labels:
          - __meta_kubernetes_service_name
        target_label: service
      - source_labels:
          - __meta_kubernetes_service_name
        target_label: job
        replacement: ${1}
      - target_label: endpoint
        replacement: "8080"
      - source_labels:
          - __meta_kubernetes_pod_name
        target_label: pod
{% endfor %}
'''

t = Template(pod_template)
s = t.render(pod_count=1000)
ep = Template(endpoint_template)
eps = ep.render(ep_count=500,pod_count=2)
sv = Template(service_template)
svs = sv.render(service_count=500)

sc = Template(scrape_config)
svr = sc.render(service_count=500)

out_dir = sys.argv[1]
config_out_dir = '/tmp'

with open(os.path.join(out_dir, 'pods.yaml'),'w+') as fin:
    fin.write(s)

with open(os.path.join(out_dir, 'endpoints.yaml'),'w+') as fin:
    fin.write(eps)

with open( os.path.join(out_dir, 'services.yaml'),'w+') as fin:
    fin.write(svs)

with open(os.path.join(config_out_dir, 'vmagent_debug.yaml'),'w+') as fin:
    fin.write(svr)