feedback experiment

biased-hard.csv

biased_start.csv

feedback.py

import numpy as np
import pandas as pd
import matplotlib.pylab as plt
from sklearn.pipeline import make_pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import recall_score, precision_score, accuracy_score
from sklego.preprocessing import ColumnSelector
from prettytable import PrettyTable


class Experiment():
    def __init__(self, estimator, biased_data, actual_data, seed=42):
        self.estimator = estimator
        self.x_train = biased_data[['x1', 'x2']].values
        self.y_train = biased_data['y'].values
        self.seed = seed
        self.x_actual = actual_data[['x1', 'x2']].values
        self.y_actual = actual_data['y'].values
        self.round = 0 
        self.logs = None
        self.estimator.fit(self.x_train, self.y_train)
    
    def retrain(self):
        learned = self.logs.loc[lambda d: d['pred'] == 1]
        X = np.concatenate([self.x_train, learned[['x1', 'x2']].values])
        y = np.concatenate([self.y_train, learned['true'].values])
        self.estimator.fit(X, y)
    
    def scores(self):
        learned = self.logs.loc[lambda d: d['pred'] == 1]
        X = np.concatenate([self.x_train, learned[['x1', 'x2']].values])
        y = np.concatenate([self.y_train, learned['true'].values])
        print(f"measured scores on seen data; acc={accuracy_score(y, self.estimator.predict(X))}")
        print(f"measured scores on seen data; precision={precision_score(y, self.estimator.predict(X))}")
        print(f"measured scores on seen data; recall={recall_score(y, self.estimator.predict(X))}")
        return self.estimator.score(X, y)
    
    def step(self, n_proposal, n_select, method):
        self.round += 1
        subset_y, self.y_actual = self.y_actual[:n_proposal], self.y_actual[n_proposal:]
        subset_x, self.x_actual = self.x_actual[:n_proposal, :], self.x_actual[n_proposal:, :]
        chosen_idx, not_chosen_idx, proba = method(self, subset_x, n_select)
        df1 = pd.DataFrame({
            'x1': subset_x[chosen_idx][:, 0], 
            'x2': subset_x[chosen_idx][:, 1], 
            'pred': [1 for i in subset_y[chosen_idx]],
            'proba': proba[chosen_idx],
            'true': subset_y[chosen_idx], 
            'round': [self.round for i in subset_y[chosen_idx]], 
        })
        df2 = pd.DataFrame({
            'x1': subset_x[not_chosen_idx][:, 0], 
            'x2': subset_x[not_chosen_idx][:, 1], 
            'pred': [0 for i in subset_y[not_chosen_idx]], 
            'proba': proba[not_chosen_idx],
            'true': subset_y[not_chosen_idx], 
            'round': [self.round for i in subset_y[not_chosen_idx]], 
        })
        logz = df1.append(df2)
        self.logs = logz if self.logs is None else self.logs.append(logz)
        self.retrain()

def greedy(experiment, x, n):
    """
    est - sklearn estimator going in
    x   - data points to predict
    n   - the number of points to return
    """
    preds = experiment.estimator.predict_proba(x)
    order = np.lexsort((np.arange(len(preds)), -preds[:, 1]))
    return order[:n], order[n:], preds[:, 1]

def sploit(experiment, x, n):
    """
    everybody who scores higher than 80% is equal
    """
    preds = experiment.estimator.predict_proba(x)
    p = np.array(preds[:, 1]) + 0.001
    best_candidate = np.argmax(p)
    p[best_candidate] = 1
    order = np.random.choice(range(len(p)), p=p/p.sum(), replace=False, size=len(p))
    return order[:n], order[n:], preds[:, 1]

def plot_experiment(biased_start, other_data, size=10, selection=1, method=greedy):
    experiment = Experiment(
        estimator=KNeighborsClassifier(n_neighbors=20),
        biased_data=biased_start, 
        actual_data=other_data
    )
    
    df = pd.concat([biased_start, other_data])
    min_x1, max_x1 = df['x1'].min(), df['x1'].max()
    min_x2, max_x2 = df['x2'].min(), df['x2'].max()

    while True:
        try:
            experiment.step(size, selection, method)
        except ValueError:
            break # this just means we've exhausted the dataset
    print(f"method={method.__name__} size={size} selection={selection}")
    experiment.scores()
    print(f"measured scores on real data; acc={accuracy_score(experiment.logs['true'], experiment.logs['proba'] > 0.5)}")
    print(f"measured scores on real data; precision={precision_score(experiment.logs['true'], experiment.logs['proba'] > 0.5)}")
    print(f"measured scores on real data; recall={recall_score(experiment.logs['true'], experiment.logs['proba'] > 0.5)}")
    plt.figure(figsize=(16, 4))
    plt.subplot(141)
    plt.scatter(df['x1'], df['x2'], c=df['y'])
    plt.title("this is the original dataset")
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2);

    plt.subplot(142)
    plt.scatter(biased_start['x1'], biased_start['x2'], c=biased_start['y'])
    plt.title("this is the biased start")
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2);

    plt.subplot(143)
    plt.scatter(experiment.logs['x1'], experiment.logs['x2'], c=experiment.logs['proba']);
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2)
    plt.title("this is what the model believes");
    
    plt.subplot(144)
    plt.scatter(experiment.logs['x1'], experiment.logs['x2'], c=experiment.logs['pred']);
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2)
    plt.title("these are the selected members");

notebook.ipynb