Ceph: assert uniform object to placement group distribution using a Chi-Squared Hypothesis Test

check_distribution.py

#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import sys, json
from collections import defaultdict
from scipy.stats import chisquare

# usage: ceph pg dump_json | ./check_distribution.py

if __name__ == '__main__':
    raw = sys.stdin.read()
    stats = json.loads(raw)

    collector = defaultdict(list)

    for pg in stats['pg_stats']:
        pool, _ = pg['pgid'].split('.')
        num_objects = pg['stat_sum']['num_objects']
        collector[pool].append(num_objects)

    # if there are no objects at all, a Chi-Square Test does not make sense -- esp. re. div-by-0
    grouped = [{'id':pool_id, 'observations':observations} for pool_id,observations in collector.items() if sum(observations) != 0]
    #print(json.dumps(grouped, sort_keys=True, indent=2))

    evaluated = [{'pool': pool['id'], 'test':{k:'{:f}'.format(v) for k,v in zip(['chi_squared', 'p_value'], chisquare(pool['observations']))}} for pool in grouped]
    print(json.dumps(evaluated, sort_keys=True, indent=2))


# vim: set tabstop=4 shiftwidth=4 expandtab:

example_output.json

[
  {
    "pool": "37", 
    "test": {
      "chi_squared": "5.903030", 
      "p_value": "0.551117"
    }
  }, 
  {
    "pool": "36", 
    "test": {
      "chi_squared": "4.695652", 
      "p_value": "0.697047"
    }
  }, 
  {
    "pool": "35", 
    "test": {
      "chi_squared": "8.485834", 
      "p_value": "0.291706"
    }
  }
]

example_output_preimages.json

[
  {
    "pool": "8", 
    "test": {
      "chi_squared": "49.000000", 
      "p_value": "0.000000"
    }
  }
]