Prometheus benchmark with different backing stores

Deployment.yml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: default
data:
  prometheus.yml: |-
    global:
      scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
      evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
    scrape_configs:
      - job_name: 'prometheus'
        static_configs:
        - targets: ['localhost:9090']
      - job_name: 'node'
        static_configs:
        - targets: [192.168.1.1:9100]
          labels:
            instance: router
        - targets: [192.168.1.157:9100]
          labels:
            instance: macpro
        - targets: [10.10.10.5:9100]
          labels:
            instance: DorfMaster
        - targets: [10.10.10.10:9100]
          labels:
            instance: storchris
        - targets: [10.10.10.20:9100]
          labels:
            instance: vchris
        - targets: [10.10.10.30:9100]
          labels:
            instance: vchris2
        - targets: [10.10.10.50:9100]
          labels:
            instance: opn1
        - targets: [10.10.10.60:9100]
          labels:
            instance: opn2
      - job_name: 'speedtest'
        scrape_timeout: 1m
        scrape_interval: 15m
        static_configs:
        - targets: [192.168.1.101:9112]
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: prometheus-pvc
  namespace: default
spec:
  storageClassName: ocs-storagecluster-ceph-rbd
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
  labels:
    app: prometheus
    deploy: persistent
  name: prometheus-persistent
  namespace: default
spec:
  port:
    targetPort: 9090
  to:
    kind: Service
    name: prometheus-persistent
    weight: 100
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: prometheus
    deploy: persistent
  name: prometheus-persistent
  namespace: default
spec:
  ports:
  - port: 9090
    protocol: TCP
    targetPort: 9090
  selector:
    app: prometheus
    deploy: persistent
  type: ClusterIP
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
  labels:
    app: prometheus
    deploy: ephemeral
  name: prometheus-ephemeral
  namespace: default
spec:
  port:
    targetPort: 9090
  to:
    kind: Service
    name: prometheus-ephemeral
    weight: 100
---
apiVersion: route.openshift.io/v1
kind: Route
metadata:
  labels:
    app: prometheus
    deploy: ephemeral
  name: prometheus-ephemeral-ram
  namespace: default
spec:
  port:
    targetPort: 9090
  to:
    kind: Service
    name: prometheus-ephemeral-ram
    weight: 100
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: prometheus
    deploy: ephemeral
  name: prometheus-ephemeral
  namespace: default
spec:
  ports:
  - port: 9090
    protocol: TCP
    targetPort: 9090
  selector:
    app: prometheus
    deploy: ephemeral
  type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: prometheus
    deploy: ephemeral-ram
  name: prometheus-ephemeral-ram
  namespace: default
spec:
  ports:
  - port: 9090
    protocol: TCP
    targetPort: 9090
  selector:
    app: prometheus
    deploy: ephemeral-ram
  type: ClusterIP
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: prometheus-ephemeral
  namespace: default
  labels:
    app: prometheus
    deploy: ephemeral
spec:
  replicas: 1
  template:
    metadata:
      name: prometheus-ephemeral
      labels:
        app: prometheus
        deploy: ephemeral
    spec:
      containers:
      - name: prometheus-ephemeral
        image: prom/prometheus:latest
        args:
          - '--config.file=/etc/prometheus/prometheus.yml'
          - '--storage.tsdb.path=/prometheus'
          - '--storage.tsdb.retention.time=30d'
          - '--web.enable-admin-api'
        ports:
        - name: webui
          containerPort: 9090
        resources:
          requests:
            cpu: 500m
            memory: 500M
          limits:
            cpu: 5000m
            memory: 5000M
        volumeMounts:
        - name: config-volume
          mountPath: /etc/prometheus
        - name: tsdb-volume
          mountPath: /prometheus
      initContainers:
      - name: download-snapshot
        image: busybox
        command: ["wget","https://promtest.s3.eu-central-1.amazonaws.com/promBack.tgz", "--output-document=/download/promBack.tgz"]
        volumeMounts:
        - name: download-volume
          mountPath: /download
      - name: extract-snapshot
        image: busybox
        command: ["tar","xzvf", "/download/promBack.tgz", "--directory=/prometheus"]
        volumeMounts:
        - name: download-volume
          mountPath: /download
        - name: tsdb-volume
          mountPath: /prometheus
      volumes:
      - name: config-volume
        configMap:
          name: prometheus-config
      - name: download-volume
        emptyDir: {}
      - name: tsdb-volume
        emptyDir: {}
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: prometheus-ephemeral-ram
  namespace: default
  labels:
    app: prometheus
    deploy: ephemeral-ram
spec:
  replicas: 1
  template:
    metadata:
      name: prometheus-ephemeral-ram
      labels:
        app: prometheus
        deploy: ephemeral-ram
    spec:
      containers:
      - name: prometheus-ephemeral-ram
        image: prom/prometheus:latest
        args:
          - '--config.file=/etc/prometheus/prometheus.yml'
          - '--storage.tsdb.path=/prometheus'
          - '--storage.tsdb.retention.time=30d'
          - '--web.enable-admin-api'
        ports:
        - name: webui
          containerPort: 9090
        resources:
          requests:
            cpu: 500m
            memory: 500M
          limits:
            cpu: 5000m
            memory: 5000M
        volumeMounts:
        - name: config-volume
          mountPath: /etc/prometheus
        - name: tsdb-volume
          mountPath: /prometheus
      initContainers:
      - name: download-snapshot
        image: busybox
        command: ["wget","https://promtest.s3.eu-central-1.amazonaws.com/promBack.tgz", "--output-document=/download/promBack.tgz"]
        volumeMounts:
        - name: download-volume
          mountPath: /download
      - name: extract-snapshot
        image: busybox
        command: ["tar","xzvf", "/download/promBack.tgz", "--directory=/prometheus"]
        volumeMounts:
        - name: download-volume
          mountPath: /download
        - name: tsdb-volume
          mountPath: /prometheus
      volumes:
      - name: config-volume
        configMap:
          name: prometheus-config
      - name: download-volume
        emptyDir: {}
      - name: tsdb-volume
        emptyDir:
          medium: Memory
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: prometheus-persistent
  namespace: default
  labels:
    app: prometheus
    deploy: persistent
spec:
  replicas: 1
  template:
    metadata:
      name: prometheus--persistent
      labels:
        app: prometheus
        deploy: persistent
    spec:
      containers:
      - name: prometheus-persistent
        image: prom/prometheus:latest
        args:
          - '--config.file=/etc/prometheus/prometheus.yml'
          - '--storage.tsdb.path=/prometheus'
          - '--storage.tsdb.retention.time=30d'
          - '--web.enable-admin-api'
        ports:
        - name: webui
          containerPort: 9090
        resources:
          requests:
            cpu: 500m
            memory: 500M
          limits:
            cpu: 5000m
            memory: 5000M
        volumeMounts:
        - name: config-volume
          mountPath: /etc/prometheus
        - name: tsdb-volume
          mountPath: /prometheus
      initContainers:
      - name: download-snapshot
        image: busybox
        command: ["wget","https://promtest.s3.eu-central-1.amazonaws.com/promBack.tgz", "--output-document=/download/promBack.tgz"]
        volumeMounts:
        - name: download-volume
          mountPath: /download
      - name: extract-snapshot
        image: busybox
        command: ["tar","xzvf", "/download/promBack.tgz", "--directory=/prometheus"]
        volumeMounts:
        - name: download-volume
          mountPath: /download
        - name: tsdb-volume
          mountPath: /prometheus
      volumes:
      - name: config-volume
        configMap:
          name: prometheus-config
      - name: download-volume
        emptyDir: {}
      - name: tsdb-volume
        persistentVolumeClaim:
          claimName: prometheus-pvc
...

result-hard.log

➜  work
QUERYCMD="%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29"
TESTPATH="api/v1/query_range?query=${QUERYCMD}&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m"
oc run --rm=true -i -t bench-ephemeral --image=jordi/ab --restart=Never -- -k -c 100 -n 10000 "http://prometheus-ephemeral:9090/${TESTPATH}"
oc run --rm=true -i -t bench-ephemeral-ram --image=jordi/ab --restart=Never -- -k -c 100 -n 10000 "http://prometheus-ephemeral-ram:9090/${TESTPATH}"
oc run --rm=true -i -t bench-persistent --image=jordi/ab --restart=Never -- -k -c 100 -n 10000 "http://prometheus-persistent:9090/${TESTPATH}"

If you don't see a command prompt, try pressing enter.

^C

Server Software:
Server Hostname:        prometheus-ephemeral
Server Port:            9090

Document Path:          /api/v1/query_range?query=%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        47128 bytes

Concurrency Level:      100
Time taken for tests:   488.789 seconds
Complete requests:      402
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      18980832 bytes
HTML transferred:       18945456 bytes
Requests per second:    0.82 [#/sec] (mean)
Time per request:       121589.413 [ms] (mean)
Time per request:       1215.894 [ms] (mean, across all concurrent requests)
Transfer rate:          37.92 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    2   2.7      0      18
Processing:  4937 104707 26285.2 116197  119582
Waiting:     4925 104706 26284.9 116197  119582
Total:       4937 104709 26283.3 116197  119582

Percentage of the requests served within a certain time (ms)
  50%  116197
  66%  116700
  75%  116994
  80%  117186
  90%  117779
  95%  117983
  98%  118300
  99%  118706
 100%  119582 (longest request)
^C^C
➜  work
➜  work
➜  work
➜  work
➜  work oc delete po bench-ephemeral
pod "bench-ephemeral" deleted
➜  work
QUERYCMD="%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29"
TESTPATH="api/v1/query_range?query=${QUERYCMD}&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m"
oc run --rm=true -i -t bench-ephemeral --image=jordi/ab --restart=Never -- -k -c 100 -n 10000 "http://prometheus-ephemeral:9090/${TESTPATH}"
oc run --rm=true -i -t bench-ephemeral-ram --image=jordi/ab --restart=Never -- -k -c 100 -n 10000 "http://prometheus-ephemeral-ram:9090/${TESTPATH}"
oc run --rm=true -i -t bench-persistent --image=jordi/ab --restart=Never -- -k -c 100 -n 10000 "http://prometheus-persistent:9090/${TESTPATH}"

➜  work
QUERYCMD="%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29"
TESTPATH="api/v1/query_range?query=${QUERYCMD}&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m"
oc run --rm=true -i -t bench-ephemeral --image=jordi/ab --restart=Never -- -k -c 100 -n 600 "http://prometheus-ephemeral:9090/${TESTPATH}"
oc run --rm=true -i -t bench-ephemeral-ram --image=jordi/ab --restart=Never -- -k -c 100 -n 600 "http://prometheus-ephemeral-ram:9090/${TESTPATH}"
oc run --rm=true -i -t bench-persistent --image=jordi/ab --restart=Never -- -k -c 100 -n 600 "http://prometheus-persistent:9090/${TESTPATH}"

If you don't see a command prompt, try pressing enter.
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Finished 600 requests


Server Software:
Server Hostname:        prometheus-ephemeral
Server Port:            9090

Document Path:          /api/v1/query_range?query=%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        47128 bytes

Concurrency Level:      100
Time taken for tests:   699.177 seconds
Complete requests:      600
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      28329600 bytes
HTML transferred:       28276800 bytes
Requests per second:    0.86 [#/sec] (mean)
Time per request:       116529.551 [ms] (mean)
Time per request:       1165.296 [ms] (mean, across all concurrent requests)
Transfer rate:          39.57 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   1.5      0       6
Processing:  4954 107903 22108.1 115490  119212
Waiting:     4945 107894 22109.5 115489  119212
Total:       4956 107903 22106.9 115490  119212

Percentage of the requests served within a certain time (ms)
  50%  115490
  66%  116207
  75%  116621
  80%  116905
  90%  117488
  95%  117903
  98%  118293
  99%  118594
 100%  119212 (longest request)
pod "bench-ephemeral" deleted
If you don't see a command prompt, try pressing enter.
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Finished 600 requests


Server Software:
Server Hostname:        prometheus-ephemeral-ram
Server Port:            9090

Document Path:          /api/v1/query_range?query=%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        47128 bytes

Concurrency Level:      100
Time taken for tests:   725.612 seconds
Complete requests:      600
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      28329600 bytes
HTML transferred:       28276800 bytes
Requests per second:    0.83 [#/sec] (mean)
Time per request:       120935.289 [ms] (mean)
Time per request:       1209.353 [ms] (mean, across all concurrent requests)
Transfer rate:          38.13 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   1.5      0       6
Processing:  5007 112023 22781.7 119908  124783
Waiting:     4998 112013 22784.4 119906  124782
Total:       5009 112024 22780.5 119908  124787

Percentage of the requests served within a certain time (ms)
  50%  119908
  66%  120504
  75%  120832
  80%  121074
  90%  121675
  95%  122082
  98%  122703
  99%  123287
 100%  124787 (longest request)
pod "bench-ephemeral-ram" deleted
If you don't see a command prompt, try pressing enter.
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Finished 600 requests


Server Software:
Server Hostname:        prometheus-persistent
Server Port:            9090

Document Path:          /api/v1/query_range?query=%28%28%28count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29%29%20-%20avg%28sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%7Bmode%3D%27idle%27%7D%5B5m%5D%29%29%29%29%20%2A%20100%29%20%2F%20count%28count%28node_cpu_seconds_total%29%20by%20%28cpu%29%29&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        47128 bytes

Concurrency Level:      100
Time taken for tests:   776.480 seconds
Complete requests:      600
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      28329600 bytes
HTML transferred:       28276800 bytes
Requests per second:    0.77 [#/sec] (mean)
Time per request:       129413.378 [ms] (mean)
Time per request:       1294.134 [ms] (mean, across all concurrent requests)
Transfer rate:          35.63 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   1.8      0      11
Processing:  5052 119938 24058.2 128003  134295
Waiting:     5043 119930 24062.6 128003  134295
Total:       5054 119939 24056.9 128004  134295

Percentage of the requests served within a certain time (ms)
  50%  128004
  66%  128905
  75%  129408
  80%  129795
  90%  130712
  95%  131423
  98%  132024
  99%  132605
 100%  134295 (longest request)
pod "bench-persistent" deleted
➜  work

result-medium.log

QUERYCMD="sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%5B5m%5D%29%29%20%2A%20100"
TESTPATH="api/v1/query_range?query=${QUERYCMD}&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m"
oc run --rm=true -i -t bench-ephemeral --image=jordi/ab --restart=Never -- -k -c 10 -n 100 "http://prometheus-ephemeral:9090/${TESTPATH}"
oc run --rm=true -i -t bench-ephemeral-ram --image=jordi/ab --restart=Never -- -k -c 10 -n 100 "http://prometheus-ephemeral-ram:9090/${TESTPATH}"
oc run --rm=true -i -t bench-persistent --image=jordi/ab --restart=Never -- -k -c 10 -n 100 "http://prometheus-persistent:9090/${TESTPATH}"

If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: unable to upgrade connection: container bench-ephemeral not found in pod bench-ephemeral_default
This is ApacheBench, Version 2.3 <$Revision: 1826891 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking prometheus-ephemeral (be patient).....done


Server Software:
Server Hostname:        prometheus-ephemeral
Server Port:            9090

Document Path:          /api/v1/query_range?query=sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%5B5m%5D%29%29%20%2A%20100&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        422665 bytes

Concurrency Level:      10
Time taken for tests:   56.956 seconds
Complete requests:      100
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      42275300 bytes
HTML transferred:       42266500 bytes
Requests per second:    1.76 [#/sec] (mean)
Time per request:       5695.644 [ms] (mean)
Time per request:       569.564 [ms] (mean, across all concurrent requests)
Transfer rate:          724.84 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    0   1.0      0      10
Processing:  2432 5417 375.7   5484    5737
Waiting:     2427 5415 375.7   5481    5736
Total:       2432 5417 375.7   5484    5737

Percentage of the requests served within a certain time (ms)
  50%   5484
  66%   5517
  75%   5605
  80%   5612
  90%   5686
  95%   5705
  98%   5723
  99%   5737
 100%   5737 (longest request)
pod "bench-ephemeral" deleted
If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: unable to upgrade connection: container bench-ephemeral-ram not found in pod bench-ephemeral-ram_default
This is ApacheBench, Version 2.3 <$Revision: 1826891 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking prometheus-ephemeral-ram (be patient).....done


Server Software:
Server Hostname:        prometheus-ephemeral-ram
Server Port:            9090

Document Path:          /api/v1/query_range?query=sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%5B5m%5D%29%29%20%2A%20100&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        422665 bytes

Concurrency Level:      10
Time taken for tests:   59.002 seconds
Complete requests:      100
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      42275300 bytes
HTML transferred:       42266500 bytes
Requests per second:    1.69 [#/sec] (mean)
Time per request:       5900.211 [ms] (mean)
Time per request:       590.021 [ms] (mean, across all concurrent requests)
Transfer rate:          699.71 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    0   1.7      0      17
Processing:  2445 5614 429.5   5618    6271
Waiting:     2444 5611 429.6   5617    6269
Total:       2446 5615 429.5   5618    6271

Percentage of the requests served within a certain time (ms)
  50%   5618
  66%   5719
  75%   5803
  80%   5892
  90%   6045
  95%   6136
  98%   6161
  99%   6271
 100%   6271 (longest request)
pod "bench-ephemeral-ram" deleted
If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: unable to upgrade connection: container bench-persistent not found in pod bench-persistent_default
This is ApacheBench, Version 2.3 <$Revision: 1826891 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking prometheus-persistent (be patient).....done


Server Software:
Server Hostname:        prometheus-persistent
Server Port:            9090

Document Path:          /api/v1/query_range?query=sum%20by%20%28mode%29%28irate%28node_cpu_seconds_total%5B5m%5D%29%29%20%2A%20100&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        422665 bytes

Concurrency Level:      10
Time taken for tests:   60.455 seconds
Complete requests:      100
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      42275300 bytes
HTML transferred:       42266500 bytes
Requests per second:    1.65 [#/sec] (mean)
Time per request:       6045.550 [ms] (mean)
Time per request:       604.555 [ms] (mean, across all concurrent requests)
Transfer rate:          682.89 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    0   0.3      0       2
Processing:  2428 5757 446.1   5823    6178
Waiting:     2426 5753 446.9   5818    6177
Total:       2429 5757 446.1   5823    6178

Percentage of the requests served within a certain time (ms)
  50%   5823
  66%   5914
  75%   5961
  80%   6020
  90%   6098
  95%   6131
  98%   6165
  99%   6178
 100%   6178 (longest request)
pod "bench-persistent" deleted

result-simple.log

QUERYCMD="node_load1"
TESTPATH="api/v1/query_range?query=${QUERYCMD}&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m"
oc run --rm=true -i -t bench-ephemeral --image=jordi/ab --restart=Never -- -k -c 100 -n 1000 "http://prometheus-ephemeral:9090/${TESTPATH}"
oc run --rm=true -i -t bench-ephemeral-ram --image=jordi/ab --restart=Never -- -k -c 100 -n 1000 "http://prometheus-ephemeral-ram:9090/${TESTPATH}"
oc run --rm=true -i -t bench-persistent --image=jordi/ab --restart=Never -- -k -c 100 -n 1000 "http://prometheus-persistent:9090/${TESTPATH}"

If you don't see a command prompt, try pressing enter.
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Completed 700 requests
Completed 800 requests
Completed 900 requests
Completed 1000 requests
Finished 1000 requests


Server Software:
Server Hostname:        prometheus-ephemeral
Server Port:            9090

Document Path:          /api/v1/query_range?query=node_load1&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        174896 bytes

Concurrency Level:      100
Time taken for tests:   14.004 seconds
Complete requests:      1000
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      174984000 bytes
HTML transferred:       174896000 bytes
Requests per second:    71.41 [#/sec] (mean)
Time per request:       1400.379 [ms] (mean)
Time per request:       14.004 [ms] (mean, across all concurrent requests)
Transfer rate:          12202.61 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   1.8      0       8
Processing:    66 1338 239.8   1388    1730
Waiting:       54 1335 240.3   1387    1730
Total:         67 1338 238.6   1388    1730

Percentage of the requests served within a certain time (ms)
  50%   1388
  66%   1410
  75%   1433
  80%   1480
  90%   1508
  95%   1589
  98%   1659
  99%   1675
 100%   1730 (longest request)
pod "bench-ephemeral" deleted
If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: unable to upgrade connection: container bench-ephemeral-ram not found in pod bench-ephemeral-ram_default
This is ApacheBench, Version 2.3 <$Revision: 1826891 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking prometheus-ephemeral-ram (be patient)
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Completed 700 requests
Completed 800 requests
Completed 900 requests
Completed 1000 requests
Finished 1000 requests


Server Software:
Server Hostname:        prometheus-ephemeral-ram
Server Port:            9090

Document Path:          /api/v1/query_range?query=node_load1&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        174896 bytes

Concurrency Level:      100
Time taken for tests:   14.148 seconds
Complete requests:      1000
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      174984000 bytes
HTML transferred:       174896000 bytes
Requests per second:    70.68 [#/sec] (mean)
Time per request:       1414.838 [ms] (mean)
Time per request:       14.148 [ms] (mean, across all concurrent requests)
Transfer rate:          12077.90 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   1.8      0       8
Processing:    70 1351 238.6   1396    1775
Waiting:       58 1348 238.1   1394    1768
Total:         70 1351 237.4   1396    1775

Percentage of the requests served within a certain time (ms)
  50%   1396
  66%   1415
  75%   1477
  80%   1486
  90%   1512
  95%   1581
  98%   1606
  99%   1675
 100%   1775 (longest request)
pod "bench-ephemeral-ram" deleted
If you don't see a command prompt, try pressing enter.
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Completed 700 requests
Completed 800 requests
Completed 900 requests
Completed 1000 requests
Finished 1000 requests


Server Software:
Server Hostname:        prometheus-persistent
Server Port:            9090

Document Path:          /api/v1/query_range?query=node_load1&start=2019-12-01T20:10:30.781Z&end=2019-12-10T20:11:00.781Z&step=10m
Document Length:        174896 bytes

Concurrency Level:      100
Time taken for tests:   15.332 seconds
Complete requests:      1000
Failed requests:        0
Keep-Alive requests:    0
Total transferred:      174984000 bytes
HTML transferred:       174896000 bytes
Requests per second:    65.22 [#/sec] (mean)
Time per request:       1533.172 [ms] (mean)
Time per request:       15.332 [ms] (mean, across all concurrent requests)
Transfer rate:          11145.71 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   1.7      0       8
Processing:    75 1463 244.7   1503    1898
Waiting:       64 1461 245.2   1502    1890
Total:         76 1464 243.5   1504    1898

Percentage of the requests served within a certain time (ms)
  50%   1504
  66%   1533
  75%   1584
  80%   1592
  90%   1618
  95%   1688
  98%   1706
  99%   1774
 100%   1898 (longest request)
pod "bench-persistent" deleted