Wrapper Method for feature Selection, Forward Search

Forward_Search.py

import numpy as np 
import ipdb

class KFoldXV():
    def __init__(self, folds=5) :
        self.folds = folds
    """
    Input :
    -----
    data : input dataset as tupple (X,y)
    model : the model used on the Data
    
    loss : the loss function 
    folds : the number of k
    
    Return :
    ------
    """
    def accuracy(self, y, y_hat):
        count=0
        for i in range(len(y)):
            if y[i]==y_hat[i]:
                count+=1
        return count/len(y)
    
    def fit(self, data, model):
        X = data[0]
        X_ = X.copy()
        y = data[1]
        y_ = y.copy()
        
        # Split D in to k mutaully exclusive subsets(Di), which union is D
        Di={}
        loss = []
        k_m = int(np.floor(X.shape[0]/self.folds))
        for i in range(self.folds-1):
            rand = np.random.choice(X_.shape[0],k_m, replace=False)
            Di[i]=rand
            X_ = np.delete(X_, rand, 0)
        Di[self.folds-1]= np.random.choice(X_.shape[0],X_.shape[0])
        for i in range(self.folds):
            model.fit(np.delete(X, Di[i], 0), np.delete(y, Di[i], 0))
            y_hat =  model.predict(X[rand])
            loss.append(self.accuracy(y[rand], y_hat))
#             if self.regr_val == 0 :
#                 loss.append(((y[rand]-y_hat)**2).mean())
#             elif self.regr_val == 1:
#                 loss.append((y[rand] == y_hat).mean())
            
        return np.array(loss).mean()


class Forward_Search():
    def __init__(self,k):
        self.k=k
    def fit(self,X,y,model,k_fold):
        d=X.shape[1]
        f=[]
        for _ in range(self.k):
            best = 0
            best_idx = []
            ind=0
            f_i = []
            for i in range(d):
                if i not in f:
                    f_i=f.copy()
                    f_i.append(i)
                    D=k_fold.fit(X[:,f_i],y,model)
                    if D>best :
                        best = D
                        ind = i
            f.append(ind)
        return f