import numpy as np
from sklearn.metrics import average_precision_score
def chance_level_ap(n_all, n_positive, trials=1000):
return np.mean([
average_precision_score([1] * n_positive + [0] * (n_all - n_positive), np.random.permutation(n_all))
for _ in range(trials)])
import pytablewriter
def create_table(config_pairs):
writer = pytablewriter.MarkdownTableWriter()
writer.header_list = ['Num of examples', 'True positive rate', 'Chance level AP']
writer.value_matrix = [
[n_all, positive_rate, chance_level_ap(n_all, round(n_all * positive_rate))]
for n_all, positive_rate in config_pairs
]
return writer
import itertools
config_pairs = itertools.product([10, 100, 1000, 10000], [0.01, 0.1, 0.5, 0.9])
create_table(config_pairs)
/Users/akiba/.pyenv/versions/anaconda3-4.2.0/lib/python3.5/site-packages/sklearn/metrics/ranking.py:444: RuntimeWarning: invalid value encountered in true_divide
recall = tps / tps[-1]
Num of examples |
True positive rate |
Chance level AP |
10 |
0.01 |
NaN |
10 |
0.10 |
0.29244 |
10 |
0.50 |
0.60708 |
10 |
0.90 |
0.91676 |
100 |
0.01 |
0.05494 |
100 |
0.10 |
0.13603 |
100 |
0.50 |
0.52077 |
100 |
0.90 |
0.90432 |
1000 |
0.01 |
0.01598 |
1000 |
0.10 |
0.10597 |
1000 |
0.50 |
0.50254 |
1000 |
0.90 |
0.90052 |
10000 |
0.01 |
0.01068 |
10000 |
0.10 |
0.10082 |
10000 |
0.50 |
0.50057 |
10000 |
0.90 |
0.90015 |