Skip to content

Instantly share code, notes, and snippets.

@iwiwi
Created July 9, 2018 04:17
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
Star You must be signed in to star a gist
Save iwiwi/506e41ae7e30946ab77be233064df997 to your computer and use it in GitHub Desktop.
import numpy as np
from sklearn.metrics import average_precision_score

def chance_level_ap(n_all, n_positive, trials=1000):
    return np.mean([
        average_precision_score([1] * n_positive + [0] * (n_all - n_positive), np.random.permutation(n_all))
        for _ in range(trials)])
import pytablewriter

def create_table(config_pairs):
    writer = pytablewriter.MarkdownTableWriter()
    writer.header_list = ['Num of examples', 'True positive rate', 'Chance level AP']
    writer.value_matrix = [
        [n_all, positive_rate, chance_level_ap(n_all, round(n_all * positive_rate))]
        for n_all, positive_rate in config_pairs
    ]
    return writer
import itertools

config_pairs = itertools.product([10, 100, 1000, 10000], [0.01, 0.1, 0.5, 0.9])

create_table(config_pairs)
/Users/akiba/.pyenv/versions/anaconda3-4.2.0/lib/python3.5/site-packages/sklearn/metrics/ranking.py:444: RuntimeWarning: invalid value encountered in true_divide
  recall = tps / tps[-1]
Num of examples True positive rate Chance level AP
10 0.01 NaN
10 0.10 0.29244
10 0.50 0.60708
10 0.90 0.91676
100 0.01 0.05494
100 0.10 0.13603
100 0.50 0.52077
100 0.90 0.90432
1000 0.01 0.01598
1000 0.10 0.10597
1000 0.50 0.50254
1000 0.90 0.90052
10000 0.01 0.01068
10000 0.10 0.10082
10000 0.50 0.50057
10000 0.90 0.90015
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment