Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
sysbench 结果统计
import os
import re
from collections import defaultdict
def get_query_per_sec(line):
"""
>>> s = 'queries: 17493427 (58309.95 per sec.)'
>>> get_query_per_sec(s)
58309.95
"""
return float(line.split('(')[1].split(' ')[0])
def get_p95_latency(line):
"""
>>> s = "95th percentile: 2.35"
>>> get_p95_latency(s)
2.35
"""
return float(line.split(':')[1].strip())
def calc_per_dir(dirname):
"""计算每个目录的结果
每个目录有三个 sysbench 结果文件
"""
query_per_sec_sum = 0
latency_list = []
for fname in os.listdir(dirname):
if not fname.endswith('.out'):
continue
fpath = os.path.join(dirname, fname)
with open(fpath) as f:
for line in f:
line = line.strip()
if line.startswith('queries:'):
query_per_sec = get_query_per_sec(line)
query_per_sec_sum += query_per_sec
if line.startswith('95th'):
latency = get_p95_latency(line)
latency_list.append(latency)
assert len(latency_list) == 3
latency_mid = sorted(latency_list)[1]
return query_per_sec_sum, latency_mid
def get_threads_count_from_dirname(dirname):
"""获取 threads 数目"""
return int(dirname.split('_')[-3])
def get_type_from_dirname(dirname):
"""获取实验类型 -- 比如:oltp-point-select"""
return '-'.join(dirname.split('_')[1:-3])
def show(result):
"""展示结果"""
for result_type, results in result.items():
print(result_type)
results = sorted(results, key=lambda each: each[0])
for each in results:
threads = each[0]
qps, latency = each[1]
print('{} \t {:.2f} \t {}'.format(each[0], qps, latency))
def show_as_md_table(result):
"""展示结果 markdown table"""
for result_type, results in result.items():
print(result_type)
print()
results = sorted(results, key=lambda each: each[0])
print('| threads | qps | latency |')
print('| ------- | --- | ------- |')
for each in results:
threads = each[0]
qps, latency = each[1]
print('| {} | {:.2f} | {} |'.format(each[0], qps, latency))
print()
def calc_all_v21():
result = defaultdict(list)
for fname in os.listdir('.'):
if fname.startswith('2.1_oltp'):
dirname = fname
threads_count = get_threads_count_from_dirname(dirname)
result_type = get_type_from_dirname(dirname)
_result = (threads_count * 3, calc_per_dir(dirname))
result[result_type].append(_result)
show_as_md_table(result)
def calc_all_v3():
result = defaultdict(list)
for fname in os.listdir('.'):
if fname.startswith('oltp'):
dirname = fname
threads_count = get_threads_count_from_dirname(dirname)
result_type = '-'.join(dirname.split('_')[:-3])
_result = (threads_count * 3, calc_per_dir(dirname))
result[result_type].append(_result)
show(result)
if __name__ == '__main__':
calc_all_v3()
@cosven

This comment has been minimized.

Copy link
Owner Author

cosven commented Jun 28, 2019

脚本测试时的目录结构

.
├── 1561618235_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561619655_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561620205_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561620640_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561621160_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561622768_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561623595_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561624485_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561625221_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561625637_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561626368_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561626626_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561627180_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561627668_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561629772_600
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561630605_600
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561636573_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561638356_600
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561640180_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561641815_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561646139_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561646708_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561647174_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561649449_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561659600_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561660112_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 1561671584_180
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_conf.patch
├── 2.1_oltp_point_select_0050_1561672175_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_point_select_0100_1561672609_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_point_select_0200_1561673046_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_point_select_0300_1561673477_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_point_select_0400_1561673917_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_point_select_0500_1561674351_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_read_write_0050_1561674790_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_read_write_0100_1561675226_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_read_write_0200_1561675668_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_read_write_0300_1561676113_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_read_write_0400_1561676553_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_read_write_0500_1561676990_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_index_0050_1561677431_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_index_0100_1561677865_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_index_0200_1561678302_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_index_0300_1561678746_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_index_0400_1561679188_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_index_0500_1561679626_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_non_index_0050_1561680071_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_non_index_0100_1561680509_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_non_index_0200_1561680945_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_non_index_0300_1561681386_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_non_index_0400_1561681825_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_oltp_update_non_index_0500_1561682263_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── 2.1_run.sh
├── 3.0_conf.patch
├── 3.0_run.sh
├── main.py
├── max-batch-wait-time
│   ├── 1561647722_180
│   │   ├── sysbench-0.out
│   │   ├── sysbench-1.out
│   │   └── sysbench-2.out
│   ├── 1561648086_180
│   │   ├── sysbench-0.out
│   │   ├── sysbench-1.out
│   │   └── sysbench-2.out
│   └── 1561648490_180
│       ├── sysbench-0.out
│       ├── sysbench-1.out
│       └── sysbench-2.out
├── oltp_point_select_0050_1561654091_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_point_select_0100_1561654524_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_point_select_0200_1561654965_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_point_select_0300_1561655402_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_point_select_0400_1561655837_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_point_select_0500_1561656270_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_read_write_0050_1561650044_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_read_write_0100_1561650487_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_read_write_0200_1561650923_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_read_write_0300_1561651362_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_read_write_0400_1561651800_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_read_write_0500_1561652241_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_index_0050_1561656803_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_index_0100_1561657238_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_index_0200_1561657674_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_index_0300_1561658119_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_index_0400_1561658558_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_index_0500_1561658994_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_non_index_0050_1561661095_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_non_index_0100_1561661529_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_non_index_0200_1561661969_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_non_index_0300_1561662411_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_non_index_0400_1561662849_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── oltp_update_non_index_0500_1561663283_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── point_select_1561642497_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── point_select_1561642933_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── point_select_1561643370_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── point_select_1561643809_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── point_select_1561644244_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
├── point_select_1561644676_300
│   ├── sysbench-0.out
│   ├── sysbench-1.out
│   └── sysbench-2.out
└── stat.py

85 directories, 258 files
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.