coin_flipping_methodology.py

from random import seed, choice
from collections import Counter
from math import sqrt

Z = 1.9599  # 95% confidence level
seed(0)     # for reproducibility


class Sample(Counter):
    """
    A recorded set of flips for a given hat.
    """
    def __init__(self, times):
        #  It's a fair coin, as per League regulations
        super(Sample, self).__init__(choice(('H', 'T')) for _ in range(times))

    @property
    def n(self):
        return sum(self.values())

    @property
    def p(self):
        return float(self['H']) / self.n

    def __add__(self, other):
        result = super(Sample, self).__add__(other)
        result.__class__ = Sample
        return result

    def __str__(self):
        error = Z * sqrt(self.p * (1 - self.p) / self.n)
        return "%.3f%% +/- %.3f%%  in %d flips" % (self.p, error, self.n)

    def __cmp__(self, other):
        p1, n1 = self.p, self.n
        p2, n2 = other.p, other.n
        p_hat = (n1 * p1 + n2 * p2) / (n1 + n2)
        if p_hat in (0, 1):
            return 0
        z = (p1 - p2) / sqrt(p_hat * (1 - p_hat) * (1.0 / n1 + 1.0 / n2))
        if abs(z) > Z:
            return cmp(z, 0)
        return 0


### Examining my old ballcap
ballcap_sample = Sample(10000)
print "Ballcap:  %s" % ballcap_sample

### Recording flips with my tiara until I have enough data that it looks good
tiara = Sample(100)
while tiara.p < 0.5:
    tiara += Sample(100)

print "Tiara:    %s" % tiara

### Using the tiara in a tournament, how do 10 flips do?
print "In competition, heads=%(H)s tails=%(T)s" % Sample(10)

### Now off to the hat store!
current_best = tiara
found_new_best_count = 0
existing_hat_best_count = 0
no_discernable_difference_count = 0
no_discernable_difference_swap_count = 0

for hat in range(100):  # Try on 100 hats
    flips = Sample(0)
    for trial in range(1000):  # Flip up to 1000 times each
        flips += Sample(1)
        current_best += Sample(1)
        if current_best < flips:
            current_best = flips
            found_new_best_count += 1
            break
        elif current_best > flips:
            existing_hat_best_count += 1
            break
    else:
        no_discernable_difference_count += 1
        if current_best.p < flips.p:
            current_best = flips
            no_discernable_difference_swap_count += 1

print
print "In the hat store, we found:"
print "  %s times where the experiment was an improvement" % found_new_best_count
print "  %s times where the control was better than the experiment" % existing_hat_best_count
print "  %s times where there wasn't a difference, of those" % no_discernable_difference_count
print "    %s times we swapped due to a higher average heads rate" % no_discernable_difference_swap_count