Wrapper Method for feature Selection, Backward Search

Backward_Search.py

import numpy as np 
import ipdb

class KFoldXV():
    def __init__(self, folds=5) :
        self.folds = folds
    """
    Input :
    -----
    data : input dataset as tupple (X,y)
    model : the model used on the Data
    
    loss : the loss function 
    folds : the number of k
    
    Return :
    ------
    """
    def accuracy(self, y, y_hat):
        count=0
        for i in range(len(y)):
            if y[i]==y_hat[i]:
                count+=1
        return count/len(y)
    
    def fit(self, data, model):
        X = data[0]
        X_ = X.copy()
        y = data[1]
        y_ = y.copy()
        
        # Split D in to k mutaully exclusive subsets(Di), which union is D
        Di={}
        loss = []
        k_m = int(np.floor(X.shape[0]/self.folds))
        for i in range(self.folds-1):
            rand = np.random.choice(X_.shape[0],k_m, replace=False)
            Di[i]=rand
            X_ = np.delete(X_, rand, 0)
        Di[self.folds-1]= np.random.choice(X_.shape[0],X_.shape[0])
        for i in range(self.folds):
            model.fit(np.delete(X, Di[i], 0), np.delete(y, Di[i], 0))
            y_hat =  model.predict(X[rand])
            loss.append(self.accuracy(y[rand], y_hat))
#             if self.regr_val == 0 :
#                 loss.append(((y[rand]-y_hat)**2).mean())
#             elif self.regr_val == 1:
#                 loss.append((y[rand] == y_hat).mean())
            
        return np.array(loss).mean()

class Backwar_search():
    
    def __init__(self,k=2):
        self.k=k
        
    def fit(self, X, y, model, k_fold):
        f=[i for i in range(X.shape[1])]
        ind=0
        worse=2
        while(len(f)>self.k):
            for i in range(X.shape[1]):
                if i in f:
                    F_i=f.copy()
                    F_i.remove(i)
                    acc = k_fold.fit(X[:,F_i],y,model)
                    if acc<worse:
                        worse=acc
                        ind=i
            f.remove(ind)
        return f