jenkins_performance_regression_test_py__PerformanceRegressionTest_test_read.json

{
    "es_url": "https://6cde69d70bffe5f26ff40d83ec6fcb26.eu-west-1.aws.found.io:9243", 
    "test_duration": 5, 
    "cluster_backend": "aws", 
    "user_prefix": "andrei-perf-regression-jks", 
    "instance_type_monitor": "t2.small", 
    "update_db_binary": "", 
    "nemesis_class_name": "ChaosMonkey", 
    "nemesis_interval": 15, 
    "region_name": "us-east-1", 
    "n_monitor_nodes": 1, 
    "time_completed": "2017-03-07 12:05", 
    "stats": [
        {
            "latency max": "254.4", 
            "keyspace_idx": "1", 
            "latency median": "6.4", 
            "stdev gc time(ms)": "0", 
            "op rate": "11142", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "11142", 
            "latency 99.9th percentile": "39.2", 
            "row rate": "11142", 
            "loader_idx": "0", 
            "total gc time (s)": "0", 
            "latency mean": "8.2", 
            "total gc mb": "0", 
            "Total partitions": "3343060", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "16.1", 
            "cpu_idx": "0", 
            "latency 99th percentile": "25.4", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "287.2", 
            "keyspace_idx": "1", 
            "latency median": "4.9", 
            "stdev gc time(ms)": "0", 
            "op rate": "13265", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "13265", 
            "latency 99.9th percentile": "45.4", 
            "row rate": "13265", 
            "loader_idx": "0", 
            "total gc time (s)": "0", 
            "latency mean": "6.9", 
            "total gc mb": "0", 
            "Total partitions": "3979529", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "15.0", 
            "cpu_idx": "1", 
            "latency 99th percentile": "24.8", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "210.9", 
            "keyspace_idx": "1", 
            "latency median": "5.7", 
            "stdev gc time(ms)": "0", 
            "op rate": "11698", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "11698", 
            "latency 99.9th percentile": "41.2", 
            "row rate": "11698", 
            "loader_idx": "1", 
            "total gc time (s)": "0", 
            "latency mean": "7.8", 
            "total gc mb": "0", 
            "Total partitions": "3509735", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "16.9", 
            "cpu_idx": "0", 
            "latency 99th percentile": "26.0", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "387.8", 
            "keyspace_idx": "1", 
            "latency median": "4.6", 
            "stdev gc time(ms)": "0", 
            "op rate": "13433", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "13433", 
            "latency 99.9th percentile": "38.9", 
            "row rate": "13433", 
            "loader_idx": "1", 
            "total gc time (s)": "0", 
            "latency mean": "6.8", 
            "total gc mb": "0", 
            "Total partitions": "4029477", 
            "Total operation time": "00:04:59", 
            "latency 95th percentile": "15.5", 
            "cpu_idx": "1", 
            "latency 99th percentile": "24.8", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "217.3", 
            "keyspace_idx": "1", 
            "latency median": "6.0", 
            "stdev gc time(ms)": "0", 
            "op rate": "11240", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "11240", 
            "latency 99.9th percentile": "39.8", 
            "row rate": "11240", 
            "loader_idx": "2", 
            "total gc time (s)": "0", 
            "latency mean": "8.2", 
            "total gc mb": "0", 
            "Total partitions": "3372034", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "17.6", 
            "cpu_idx": "0", 
            "latency 99th percentile": "26.9", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "297.3", 
            "keyspace_idx": "1", 
            "latency median": "3.9", 
            "stdev gc time(ms)": "0", 
            "op rate": "14618", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "14618", 
            "latency 99.9th percentile": "39.9", 
            "row rate": "14618", 
            "loader_idx": "2", 
            "total gc time (s)": "0", 
            "latency mean": "6.3", 
            "total gc mb": "0", 
            "Total partitions": "4385466", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "15.8", 
            "cpu_idx": "1", 
            "latency 99th percentile": "25.0", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "213.4", 
            "keyspace_idx": "1", 
            "latency median": "5.9", 
            "stdev gc time(ms)": "0", 
            "op rate": "11567", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "11567", 
            "latency 99.9th percentile": "39.1", 
            "row rate": "11567", 
            "loader_idx": "3", 
            "total gc time (s)": "0", 
            "latency mean": "7.9", 
            "total gc mb": "0", 
            "Total partitions": "3470129", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "17.2", 
            "cpu_idx": "0", 
            "latency 99th percentile": "26.3", 
            "avg gc time(ms)": "NaN"
        }, 
        {
            "latency max": "271.2", 
            "keyspace_idx": "1", 
            "latency median": "3.5", 
            "stdev gc time(ms)": "0", 
            "op rate": "17337", 
            "Total errors": "0", 
            "total gc count": "0", 
            "partition rate": "17337", 
            "latency 99.9th percentile": "39.6", 
            "row rate": "17337", 
            "loader_idx": "3", 
            "total gc time (s)": "0", 
            "latency mean": "5.3", 
            "total gc mb": "0", 
            "Total partitions": "5201165", 
            "Total operation time": "00:05:00", 
            "latency 95th percentile": "13.9", 
            "cpu_idx": "1", 
            "latency 99th percentile": "24.0", 
            "avg gc time(ms)": "NaN"
        }
    ], 
    "subnet_id": "subnet-ec4a72c4", 
    "cassandra-stress read": {
        "errors": "ignore", 
        "cl": "QUORUM", 
        "pop": "'dist=gauss(1..300000,150000,15000)'", 
        "rate": "threads=100", 
        "mode": "cql3", 
        "port": "jmx=6868", 
        "schema": "'replication(factor=3)'"
    }, 
    "ami_loader_user": "centos", 
    "es_user": "elastic", 
    "test_name": "performance_regression_test.py:PerformanceRegressionTest.test_read", 
    "ami_id_monitor": "ami-c59ac0d2", 
    "ami_id_db_scylla": "ami-c59ac0d2", 
    "user_credentials_path": "", 
    "ami_id_db_scylla_desc": "1.4.0", 
    "stress_modes": "write", 
    "space_node_threshold": 6442, 
    "ami_id_loader": "ami-c59ac0d2", 
    "ami_db_cassandra_user": "ubuntu", 
    "n_db_nodes": 3, 
    "ami_db_scylla_user": "centos", 
    "ami_monitor_user": "centos", 
    "security_group_ids": "sg-c5e1f7a0", 
    "instance_type_db": "i2.2xlarge", 
    "instance_type_loader": "c4.large", 
    "n_loaders": 4, 
    "db_type": "scylla", 
    "es_password": "******", 
    "cassandra-stress write": {
        "errors": "ignore", 
        "cl": "QUORUM", 
        "pop": "seq=1..300000", 
        "rate": "threads=100", 
        "mode": "cql3", 
        "port": "jmx=6868", 
        "schema": "'replication(factor=3)'"
    }, 
    "failure_post_behavior": "destroy"
}

The test params looks much better like this, few comments:

1. The example here is for test read, however the stress_mode described below is "write".
   This happens because before every read/mixed test we must use "write" as a pre-step.
   The write parameters are important as well, so we need to find a way to store them both.

2. Please chagne "rate": "threads=100" to "rate threads:" "100".

3. For the stats (results part), IIUC each section here is "per-node" result.
   Can you please add a "total" part that will average each stat across all nodes results?

1. hm, as I see there is no any 'write' operations before test. all write/read commands are used in tests directly
2. fixed
3. here is example with average stats https://gist.github.com/abvgedeika/542802d74809ae9203a3a6944d64a300