Cosbench misc parses

cosb_grafa_link.py

# This program parses the cosbench run-histpry CSV file and produces a HTML output
# with links to Grafana at scrolled to the time when the cosbench tests were executed

#!/usr/bin/env python
import csv
import datetime
import sys

with open(sys.argv[0]) as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    line_count = 0
    for row in csv_reader:
      if line_count >0 and len(row[3]) > 0 and len(row[4]) > 0:
        # Calculate unix timestamp and convert to ms
        start_time = int((datetime.datetime.strptime(row[3], '%Y-%m-%d %H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds() * 1000) - 60000
        stop_time = int((datetime.datetime.strptime(row[4], '%Y-%m-%d %H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds() * 1000) + 60000
        start_data = {"time": start_time, "text": 'START {}-{}'.format(row[0], row[1]), "tags": ["gosbench", "bench"] }
        end_data = {"time": stop_time, "text": 'STOP {}-{}'.format(row[0], row[1]), "tags": ["gosbench", "bench"] }
        print('{}-{} <a href="http://100.67.138.59/d/R67SuKSZk/cblum-overview?from={}&to={}"> overview </a></br>'.format(row[0],row[1],start_time,stop_time))
      line_count+=1

cosb_obj_count.py

# This program parses the cosbench run-histpry CSV file and identifies Cosbench test start and stop times.
# It then connects to a Prometheus TSDB and tries to find the ceph_cluster_total_objects at these times.
# The results are then written to stdout in CSV format

# Make sure to update the Prometheus server's address below

#!/usr/bin/env python3
import csv
import datetime
import sys
import requests

print('workload ID;workload Description;objects before workload;objects after workload')


with open(sys.argv[0]) as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    line_count = 0
    for row in csv_reader:
      if line_count >0 and len(row[3]) > 0 and len(row[4]) > 0:
        # Calculate unix timestamp and convert to ms
        start_time = int((datetime.datetime.strptime(row[3], '%Y-%m-%d %H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds())
        stop_time = int((datetime.datetime.strptime(row[4], '%Y-%m-%d %H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds())
        object_count_start = 0
        object_count_stop = 0

        payload = {
          'query': 'ceph_cluster_total_objects',
          'time': start_time
        }
        r = requests.get('http://192.168.120.9:9090/api/v1/query', params=payload)
        if r.status_code == requests.codes.ok:
          try:
            object_count_start = r.json()['data']['result'][0]['value'][1]
          except (KeyError, IndexError):
            pass
        payload = {
          'query': 'ceph_cluster_total_objects',
          'time': stop_time
        }
        r = requests.get('http://192.168.120.9:9090/api/v1/query', params=payload)
        if r.status_code == requests.codes.ok:
          try:
            object_count_stop = r.json()['data']['result'][0]['value'][1]
          except (KeyError, IndexError):
            pass

        print('{};{};{};{}'.format(row[0],row[1],object_count_start, object_count_stop))
      line_count+=1

cosb_parse_run-hist.py

# This program processes the cosbench run-histoy CSV file and
# inserts Grafana annotations where necessary
# This marks cosbench start and stop times in Grafana graphs

# Make sure to filter for the 'bench' tag in Grafana to see the annotations in your dashboards

#!/usr/bin/env python
import csv
import datetime
import sys
import requests

token = "eyJrIjoiUnNkYmZyTFE4MlpUNDNHYWlZVVFnYmNzQ3pWdHJiNTMiLCJuIjoiYW5ub3RhdG9yIiwiaWQiOjF9" # new Grafana
# token = "eyJrIjoiaHJ0NmZEZXBvc3IxMkhYQWN6c1ZuYTJCZDNsbFdqMFgiLCJuIjoiYW5ub3RhdG9yIiwiaWQiOjF9" # old Grafana
headers = {"Authorization": "Bearer {}".format(token)}
endpoint = "http://192.168.120.9:4000/api/annotations"
get_params = {"type": "annotation"}

annotations = requests.get(endpoint, headers=headers, params=get_params).json()


with open(sys.argv[0]) as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    line_count = 0
    for row in csv_reader:
      if line_count >0 and len(row[3]) > 0 and len(row[4]) > 0:
        # Calculate unix timestamp and convert to ms
        start_time = int((datetime.datetime.strptime(row[3], '%Y-%m-%d %H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds() * 1000)
        stop_time = int((datetime.datetime.strptime(row[4], '%Y-%m-%d %H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds() * 1000)
        print('{} - {} Name: {}-{}').format(start_time, stop_time, row[0], row[1])
        start_data = {"time": start_time, "text": 'START {}-{}'.format(row[0], row[1]), "tags": ["gosbench", "bench"] }
        end_data = {"time": stop_time, "text": 'STOP {}-{}'.format(row[0], row[1]), "tags": ["gosbench", "bench"] }
        start = requests.post(endpoint, headers=headers, data=start_data)
        stop = requests.post(endpoint, headers=headers, data=end_data)
        if not start or not stop:
          print("Issue with generating annotation for {}".format(row[0]))
          sys.exit(1)
      line_count+=1