feedback experiment

biased-hard.csv

biased_start.csv

feedback.py

import numpy as np
import pandas as pd
import matplotlib.pylab as plt
from sklearn.pipeline import make_pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import recall_score, precision_score, accuracy_score
from sklego.preprocessing import ColumnSelector
from prettytable import PrettyTable


class Experiment():
    def __init__(self, estimator, biased_data, actual_data, seed=42):
        self.estimator = estimator
        self.x_train = biased_data[['x1', 'x2']].values
        self.y_train = biased_data['y'].values
        self.seed = seed
        self.x_actual = actual_data[['x1', 'x2']].values
        self.y_actual = actual_data['y'].values
        self.round = 0 
        self.logs = None
        self.estimator.fit(self.x_train, self.y_train)
    
    def retrain(self):
        learned = self.logs.loc[lambda d: d['pred'] == 1]
        X = np.concatenate([self.x_train, learned[['x1', 'x2']].values])
        y = np.concatenate([self.y_train, learned['true'].values])
        self.estimator.fit(X, y)
    
    def scores(self):
        learned = self.logs.loc[lambda d: d['pred'] == 1]
        X = np.concatenate([self.x_train, learned[['x1', 'x2']].values])
        y = np.concatenate([self.y_train, learned['true'].values])
        print(f"measured scores on seen data; acc={accuracy_score(y, self.estimator.predict(X))}")
        print(f"measured scores on seen data; precision={precision_score(y, self.estimator.predict(X))}")
        print(f"measured scores on seen data; recall={recall_score(y, self.estimator.predict(X))}")
        return self.estimator.score(X, y)
    
    def step(self, n_proposal, n_select, method):
        self.round += 1
        subset_y, self.y_actual = self.y_actual[:n_proposal], self.y_actual[n_proposal:]
        subset_x, self.x_actual = self.x_actual[:n_proposal, :], self.x_actual[n_proposal:, :]
        chosen_idx, not_chosen_idx, proba = method(self, subset_x, n_select)
        df1 = pd.DataFrame({
            'x1': subset_x[chosen_idx][:, 0], 
            'x2': subset_x[chosen_idx][:, 1], 
            'pred': [1 for i in subset_y[chosen_idx]],
            'proba': proba[chosen_idx],
            'true': subset_y[chosen_idx], 
            'round': [self.round for i in subset_y[chosen_idx]], 
        })
        df2 = pd.DataFrame({
            'x1': subset_x[not_chosen_idx][:, 0], 
            'x2': subset_x[not_chosen_idx][:, 1], 
            'pred': [0 for i in subset_y[not_chosen_idx]], 
            'proba': proba[not_chosen_idx],
            'true': subset_y[not_chosen_idx], 
            'round': [self.round for i in subset_y[not_chosen_idx]], 
        })
        logz = df1.append(df2)
        self.logs = logz if self.logs is None else self.logs.append(logz)
        self.retrain()

def greedy(experiment, x, n):
    """
    est - sklearn estimator going in
    x   - data points to predict
    n   - the number of points to return
    """
    preds = experiment.estimator.predict_proba(x)
    order = np.lexsort((np.arange(len(preds)), -preds[:, 1]))
    return order[:n], order[n:], preds[:, 1]

def sploit(experiment, x, n):
    """
    everybody who scores higher than 80% is equal
    """
    preds = experiment.estimator.predict_proba(x)
    p = np.array(preds[:, 1]) + 0.001
    best_candidate = np.argmax(p)
    p[best_candidate] = 1
    order = np.random.choice(range(len(p)), p=p/p.sum(), replace=False, size=len(p))
    return order[:n], order[n:], preds[:, 1]

def plot_experiment(biased_start, other_data, size=10, selection=1, method=greedy):
    experiment = Experiment(
        estimator=KNeighborsClassifier(n_neighbors=20),
        biased_data=biased_start, 
        actual_data=other_data
    )
    
    df = pd.concat([biased_start, other_data])
    min_x1, max_x1 = df['x1'].min(), df['x1'].max()
    min_x2, max_x2 = df['x2'].min(), df['x2'].max()

    while True:
        try:
            experiment.step(size, selection, method)
        except ValueError:
            break # this just means we've exhausted the dataset
    print(f"method={method.__name__} size={size} selection={selection}")
    experiment.scores()
    print(f"measured scores on real data; acc={accuracy_score(experiment.logs['true'], experiment.logs['proba'] > 0.5)}")
    print(f"measured scores on real data; precision={precision_score(experiment.logs['true'], experiment.logs['proba'] > 0.5)}")
    print(f"measured scores on real data; recall={recall_score(experiment.logs['true'], experiment.logs['proba'] > 0.5)}")
    plt.figure(figsize=(16, 4))
    plt.subplot(141)
    plt.scatter(df['x1'], df['x2'], c=df['y'])
    plt.title("this is the original dataset")
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2);

    plt.subplot(142)
    plt.scatter(biased_start['x1'], biased_start['x2'], c=biased_start['y'])
    plt.title("this is the biased start")
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2);

    plt.subplot(143)
    plt.scatter(experiment.logs['x1'], experiment.logs['x2'], c=experiment.logs['proba']);
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2)
    plt.title("this is what the model believes");
    
    plt.subplot(144)
    plt.scatter(experiment.logs['x1'], experiment.logs['x2'], c=experiment.logs['pred']);
    plt.xlim(min_x1, max_x1)
    plt.ylim(min_x2, max_x2)
    plt.title("these are the selected members");

notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pylab as plt\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklego.preprocessing import ColumnSelector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"biased-hard.csv\")\n",
    "# df = pd.read_csv(\"biased_start.csv\")\n",
    "df.columns = [\"x1\", \"x2\", \"y\"]\n",
    "df[\"y\"] = (df[\"y\"] == \"b\").astype(np.int)\n",
    "plt.scatter(df['x1'], df['x2'], c=df['y']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(41)\n",
    "biased_start = df.loc[lambda d: d['x1'] > 450].sample(30) #biased-hard\n",
    "# biased_start = df.loc[lambda d: d['x1'] > 250].sample(30) # biased_start\n",
    "other_data = (df\n",
    "              .reset_index()\n",
    "              .loc[lambda d: ~d['index'].isin(biased_start.index)]\n",
    "              .drop(columns=\"index\")\n",
    "              .sample(df.shape[0] - biased_start.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_x1, max_x1 = df['x1'].min(), df['x1'].max()\n",
    "min_x2, max_x2 = df['x2'].min(), df['x2'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(16, 4))\n",
    "plt.subplot(131)\n",
    "plt.scatter(df['x1'], df['x2'], c=df['y'])\n",
    "plt.title(\"this is the whole dataset\")\n",
    "plt.xlim(min_x1, max_x1)\n",
    "plt.ylim(min_x2, max_x2);\n",
    "\n",
    "plt.subplot(132)\n",
    "plt.scatter(biased_start['x1'], biased_start['x2'], c=biased_start['y'])\n",
    "plt.title(\"this is the biased start\")\n",
    "plt.xlim(min_x1, max_x1)\n",
    "plt.ylim(min_x2, max_x2);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What will happen?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from feedback import Experiment, plot_experiment, greedy, sploit\n",
    "from scipy.spatial import distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_experiment(biased_start=biased_start, other_data=other_data, size=10, selection=1, method=greedy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_experiment(biased_start=biased_start, other_data=other_data, size=10, selection=2, method=greedy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}