Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
LSTM convergence optimization
Loss at epoch 0 = tensor(0.6932, grad_fn=<NllLossBackward>)
Loss at epoch 1000 = tensor(0.5234, grad_fn=<NllLossBackward>)
Loss at epoch 2000 = tensor(0.5206, grad_fn=<NllLossBackward>)
Loss at epoch 3000 = tensor(0.3477, grad_fn=<NllLossBackward>)
Loss at epoch 4000 = tensor(0.3470, grad_fn=<NllLossBackward>)
Loss at epoch 5000 = tensor(0.3468, grad_fn=<NllLossBackward>)
Loss at epoch 6000 = tensor(0.3467, grad_fn=<NllLossBackward>)
Loss at epoch 7000 = tensor(0.3466, grad_fn=<NllLossBackward>)
Loss at epoch 8000 = tensor(0.3466, grad_fn=<NllLossBackward>)
Loss at epoch 9000 = tensor(0.3466, grad_fn=<NllLossBackward>)
tensor([[[-6.9315e-01, -6.9315e-01],
[-6.9315e-01, -6.9315e-01]],
[[-5.3881e-05, -1.0168e+01],
[-9.8277e+00, -3.8385e-05]],
[[ 0.0000e+00, -1.8239e-05],
[-2.3094e+01, -1.0910e+01]],
[[-3.5763e-06, 0.0000e+00],
[-1.2558e+01, -2.6699e+01]]], grad_fn=<LogSoftmaxBackward>) tensor([[0, 0],
[0, 1],
[1, 1],
[0, 0]])
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import numpy as np
#from fastai.learner import *
torch.manual_seed(1)
#torch.cuda.set_device(0)
bs = 2
x_trn = torch.tensor([[1.0000, 1.0000],
[1.0000, 0.9870],
[0.9962, 0.9848],
[1.0000, 1.0000]])#.cuda()
y_trn = torch.tensor([[0, 0],
[0, 1],
[1, 1],
[0, 0]])#.cuda()
n_hidden = 5
n_classes = 2
class TESS_LSTM(nn.Module):
def __init__(self, nl):
super().__init__()
self.nl = nl
self.rnn = nn.LSTM(1, n_hidden, nl)
self.l_out = nn.Linear(n_hidden, n_classes)
self.init_hidden(bs)
def forward(self, input):
outp,h = self.rnn(input.view(len(input), bs, -1), self.h)
return F.log_softmax(self.l_out(outp),dim=1)
def init_hidden(self, bs):
self.h = (Variable(torch.zeros(self.nl, bs, n_hidden)),Variable(torch.zeros(self.nl, bs, n_hidden)))
model = TESS_LSTM(1)#.cuda()
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
for epoch in range(10000):
model.zero_grad()
tag_scores = model(x_trn)
loss = loss_function(tag_scores.reshape(4*bs,n_classes), y_trn.reshape(4*bs))
loss.backward()
optimizer.step()
if epoch%1000==0:
print("Loss at epoch %d = " %epoch, loss)
print(model(x_trn), y_trn)
@DevMahmoud10

This comment has been minimized.

Copy link
Owner Author

@DevMahmoud10 DevMahmoud10 commented Mar 21, 2019

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment