adaboost.py

def adaboost_fit(X,y, M=10, learning_rate = 1):
    #Initialization of utility variables
    N = len(y)
    estimator_list, y_predict_list, estimator_error_list, estimator_weight_list, sample_weight_list = [], [],[],[],[]

    #Initialize the sample weights
    sample_weight = np.ones(N) / N
    sample_weight_list.append(sample_weight.copy())

    for m in range(M):   

        #Fit a classifier
        #estimator = DecisionTreeClassifier(max_depth = 1, max_leaf_nodes=2)
        #estimator.fit(X, y, sample_weight=sample_weight)
        #y_predict = estimator.predict(X)
        w, b = train_axis_aligned_lin_classifier(X,y, weights=sample_weight)
        y_predict_temp = np.sign(np.dot(X,w) + b).flatten()
        sign_test = []
        for sign in [-1, 1]:
            incorrect = (sign * y_predict_temp != y)
            sign_test.append((sign, np.sum(incorrect)))
        sign_test.sort(key=lambda x: x[1])
        sign = sign_test[0][0]
        y_predict = sign*y_predict_temp
            
        #Misclassifications
        incorrect = (y_predict != y)

        #Estimator error
        estimator_error = np.mean( np.average(incorrect, weights=sample_weight, axis=0))
        
        #Boost estimator weights
        estimator_weight =  learning_rate * np.log((1. - estimator_error) / estimator_error)

        #Boost sample weights
        sample_weight *= np.exp(estimator_weight * incorrect * ((sample_weight > 0) | (estimator_weight < 0)))

        #Save iteration values
        estimator_list.append((sign,w,b))
        #estimator_list.append(estimator)
        y_predict_list.append(y_predict.copy())
        estimator_error_list.append(estimator_error.copy())
        estimator_weight_list.append(estimator_weight.copy())
        sample_weight_list.append(sample_weight.copy())
        

    #Convert to np array for convenience   
    #estimator_list = np.array(estimator_list)
    y_predict_list = np.array(y_predict_list)
    estimator_error_list = np.array(estimator_error_list)
    estimator_weight_list = np.array(estimator_weight_list)
    sample_weight_list = np.array(sample_weight_list)

    #Predictions
    preds = (np.array([np.sign((y_predict_list[:,point] * estimator_weight_list).sum()) for point in range(N)]))
    print('Accuracy = ', (preds == y).sum() / N) 
    
    return estimator_list, estimator_weight_list, sample_weight_list
    
 estimator_list, estimator_weight_list, sample_weight_list  = adaboost_fit(X,Y, M=15, learning_rate = 1)
 
 def ada_boost_class_fun(X):
    temp_pred = np.array( [ (e[0]*np.sign(np.dot(X,e[1]) + e[2]).flatten()).T* w for e, w in zip(estimator_list,estimator_weight_list )]  ) / estimator_weight_list.sum()
    return np.sign(temp_pred.sum(axis = 0))
    
 show_results(X,Y,ada_boost_class_fun)
 
 for i in range(4):
    print('Happy Birthday to you!')