Decision_stump.py

import numpy as np
np.set_printoptions(2)

dataY = np.array([1, -1, 1, 1, 1,
                  -1, -1, 1, -1, -1])
dataX = np.arange(len(dataY))
print('X    ', dataX)
print('Y    ', dataY)


class DecisionStump():
    def __init__(self, u):
        self.a = 0
        self.b = 1
        self.u = u

    def evaluate(self, x):
        return self.b * (2 * (x < self.a) - 1)

    def loss(self):
        cond = dataY * self.evaluate(dataX) < 0
        return np.sum(self.u[cond]) / np.sum(self.u)

    def findOpt(self):
        min_loss = np.Inf
        opt = []
        for i in range(len(dataY) + 1):
            for j in range(-1, 2, 2):
                self.a = i
                self.b = j
                l = self.loss()
                if l < min_loss:
                    min_loss = l
                    opt = (i, j)
        self.a, self.b = opt
        print('error', min_loss)
        print('a, b ', opt)
        assert(min_loss <= .5)

    def getAlpha(self):
        e = self.loss()
        return np.log((1 - e) / e) / 2

    def getNextU(self):
        u = np.copy(self.u)
        alpha = self.getAlpha()
        u *= np.exp(-alpha * self.evaluate(dataX) * dataY)
        print('alpha', alpha)
        print('NEWu ', u)
        return u


u = np.ones(len(dataY))
boost = []

for i in range(3):
    print('\n', i + 1)
    f = DecisionStump(u)
    f.findOpt()
    u = f.getNextU()
    boost.append(f)

print('\n', 'All')
predict = []
for f in boost:
    predict.append(f.getAlpha() * f.evaluate(dataX))

predict = np.sum(predict, axis=0)
print('predict', predict)
print('binary', 2 * (predict >= 0) - 1)
print(dataY)

OUTPUT

X     [0 1 2 3 4 5 6 7 8 9]
Y     [ 1 -1  1  1  1 -1 -1  1 -1 -1]

 1
error 0.2
a, b  (5, 1)
alpha 0.6931471805599453
NEWu  [0.5 2.  0.5 0.5 0.5 0.5 0.5 2.  0.5 0.5]

 2
error 0.3125
a, b  (2, -1)
alpha 0.39422868018213514
NEWu  [0.74 1.35 0.34 0.34 0.34 0.74 0.74 1.35 0.74 0.74]

 3
error 0.3181818181818181
a, b  (1, 1)
alpha 0.38107002602344847
NEWu  [0.51 0.92 0.49 0.49 0.49 0.51 0.51 1.97 0.51 0.51]

 All
predict [ 0.68 -0.08  0.71  0.71  0.71 -0.68 -0.68 -0.68 -0.68 -0.68]
binary [ 1 -1  1  1  1 -1 -1 -1 -1 -1]