naive CBOW architecture

cbow_model.py

class MyCbow():
    def __init__(self, vocab_size, embedding_size):
        super(MyCbow, self).__init__()
        self.w1 = self.initialize(vocab_size, embedding_size)
        self.w2 = self.initialize(embedding_size, vocab_size)
        self.w1_grad= 0
        self.w2_grad = 0
        self.embedding = None
        self.x = None
    def set_params(self, w):
        self.w1, self.w2 = w[0], w[1]
        
        
    def initialize(self, input_size, output_size):
        w = np.zeros(shape=(input_size, output_size), dtype='float64')
        for i in range(input_size):
            for j in range(output_size):
                w[i,j] = random()
        return w
    def softmax(self,x):
        exps = np.exp(x - x.max())
        return exps / np.sum(exps)

    def forward(self, x):
        self.x = x # samples*features
        self.embedding = np.dot(self.x, self.w1) #features*hidden_size
        self.avg_embedding = np.mean(self.embedding, axis=0) #1*hidden_size
        output = np.dot(self.avg_embedding, self.w2) 
        return self.softmax(output)

    def backward(self, err, lr):
        # self.embedding = np.mean(self.embedding, axis=0) #1*hidden_size
        self.w2_grad = lr*np.dot(self.avg_embedding.reshape(-1,1), err.reshape(1,-1))    
        # hidden_size*features
        err2 = np.dot(self.w2, err.reshape(1,-1).T)
        err2 = self.embedding*(1 - self.embedding)
        self.w1_grad = lr*np.dot(self.x.T,  err2)

    def update(self):
        w1 = self.w1 + self.w1_grad 
        w2 = self.w2 + self.w2_grad
        self.set_params([w1, w2])