drnn.py

Class Net(nn.module):
  self.depth = 5
  self.hidden_width = 100

  drnn_h = [Variable(torch.zeros(2 ** i, 1, self.hidden_width)).cuda().float()
                    for i in range(self.depth)]

  self.drnn1 = drnn.DRNN(self.hidden_width, self.hidden_width, self.depth)
  ...
  ...

      def process(self, input):
          input = self.io_noise(input)
          lin_1 = self.lin1(input).view(1, 1, self.hidden_width)
          drnn_out, self.drnn_h = self.drnn1(lin_1, self.drnn_h)
          drnn_out = drnn_out.view(1, self.hidden_width)
          output = self.lin2(drnn_out).view(1, 1, self.io_width)
          return output

DRNN (5 layer, 100 wide, with memory)
Avg GPU Memory Allocation: 1102 MB
Avg Iteration time: 43.842 sec

      
    

gru.py

Class Net(nn.module):
  self.depth = 5
  self.hidden_width = 100

  self.gru1 = nn.GRU(self.hidden_width, self.hidden_width, self.depth)
  self.gru1_h = Variable(torch.zeros(self.depth, 1, self.hidden_width), requires_grad=False).cuda().float()
  ...
  ...
      def process(self, input):
          input = self.io_noise(input)
          lin_1 = self.lin1(input).view(1, 1, self.hidden_width)
          gru_out, self.gru1_h = self.gru1.forward(lin_1, self.gru1_h)
          gru_out = gru_out.view(1, self.hidden_width)
          output = self.lin2(gru_out).view(1, 1, self.io_width)
          return output
      
GRU (5 layer, 100 wide, with memory)
Avg Gpu Memory Allocation: 427 MB
Avg Ieration time: 19.35 sec