Environment

PyTorch version: 1.4.0
Is debug build: No
CUDA used to build PyTorch: 10.1

OS: Ubuntu 18.04.2 LTS
GCC version: (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0
CMake version: version 3.10.2

Python version: 3.7
Is CUDA available: Yes
CUDA runtime version: 10.1.243

Nvidia driver version: 418.67
cuDNN version: /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcudnn.so.7.6.5

Versions of relevant libraries:
[pip] numpy==1.17.3
[pip] torch==1.4.0
[pip] torchsummary==1.5.1
[pip] torchvision==0.5.0
[conda] blas                      1.0                         mkl  
[conda] mkl                       2019.4                      243  
[conda] mkl-service               2.3.0            py37he904b0f_0  
[conda] mkl_fft                   1.0.15           py37ha843d7b_0  
[conda] mkl_random                1.1.0            py37hd6b4f25_0  
[conda] pytorch                   1.4.0           py3.7_cuda10.1.243_cudnn7.6.3_0    pytorch
[conda] torchsummary              1.5.1                    pypi_0    pypi
[conda] torchvision               0.5.0                py37_cu101    pytorch

error log

Traceback (most recent call last):
  File "reproduce_error.py", line 148, in <module>
    output = model(input)
  File "/home/namle/anaconda3/envs/condapy3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "reproduce_error.py", line 83, in forward
    x_hat = self.decoder(y)
  File "/home/namle/anaconda3/envs/condapy3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "reproduce_error.py", line 68, in forward
    x = self.network(x)
  File "/home/namle/anaconda3/envs/condapy3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/namle/anaconda3/envs/condapy3/lib/python3.7/site-packages/torch/nn/modules/container.py", line 100, in forward
    input = module(input)
  File "/home/namle/anaconda3/envs/condapy3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/namle/anaconda3/envs/condapy3/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 921, in forward
    output_padding, self.groups, self.dilation)
RuntimeError: CUDA error: an illegal memory access was encountered

reproduce_pytorch_error.py

import torch
from torch import nn
import numpy as np
import torch.utils.data as data
import torchsummary
import os
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
class Encoder(nn.Module):

  def __init__(self, input_channels=3, output_channels=64, kernel_size=3):

    super(Encoder, self).__init__()
    self.input_channels = input_channels

    padding = int((kernel_size - 1)/2)
    factor = 32
    self.network = nn.Sequential(    
       nn.Conv3d(input_channels, factor * 4, kernel_size=kernel_size, stride=1, padding=padding),
       nn.BatchNorm3d(factor * 4, affine=False),
       nn.ReLU(True),

       nn.Conv3d(factor * 4, factor * 2, kernel_size=kernel_size, stride=2, padding=padding),
       nn.BatchNorm3d(factor * 2, affine=False),
       nn.ReLU(True),

       nn.Conv3d(factor * 2, factor * 2, kernel_size=kernel_size, stride=2, padding=padding),
       nn.BatchNorm3d(factor * 2, affine=False),
       nn.ReLU(True),

       nn.Conv3d(factor * 2, output_channels, kernel_size=kernel_size, stride=1, padding=padding),
       nn.BatchNorm3d(output_channels, affine=False),
       nn.ReLU(True)
    )

  def forward(self, x):
    '''overide forward function from base class
    '''
    x = self.network(x)
    return x

class Decoder(nn.Module):

  def __init__(self, input_channels=64, output_channels=3, kernel_size=3):

    super(Decoder, self).__init__()
    self.input_channels = input_channels
    self.output_channels = output_channels
    factor = 32
    padding = int((kernel_size - 1)/2)

    self.network = nn.Sequential( 
       nn.ConvTranspose3d(input_channels, factor * 2, kernel_size=kernel_size, stride=1, padding=padding),
       nn.BatchNorm3d(factor * 2, affine=False),
       nn.ReLU(True),
    
       nn.ConvTranspose3d(factor * 2, factor * 2, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1),
       nn.BatchNorm3d(factor * 2, affine=False),
       nn.ReLU(True),

       nn.ConvTranspose3d(factor * 2, factor * 4, kernel_size=kernel_size, stride=2, padding=padding, output_padding=1),
       nn.BatchNorm3d(factor * 4, affine=False),
       nn.ReLU(True),

       nn.Conv3d(factor * 4, self.output_channels, kernel_size=kernel_size, stride=1, padding=padding),
    )

  def forward(self, x):
    x = self.network(x)
    return x

class Autoencoder(nn.Module):
  '''Autoencoder model for videos
  '''

  def __init__(self, kernel_size = 3):
    super(Autoencoder, self).__init__()
    latent_dim = 4
    self.encoder = Encoder(input_channels=3, output_channels=latent_dim, kernel_size=kernel_size)
    self.decoder = Decoder(input_channels=latent_dim, output_channels=3, kernel_size=kernel_size)

  def forward(self,x):
    y = self.encoder(x)
    x_hat = self.decoder(y)
    return x_hat

class TrainDataset(data.Dataset):
    # def __init__(self):
       

    def __getitem__(self, index):
       return torch.rand(3, 16, 112, 112)

    def __len__(self):
        return 1000

class TestDataset(data.Dataset):
    # def __init__(self):
       

    def __getitem__(self, index):
       return torch.rand(3, 16, 256, 342)

    def __len__(self):
        return 1000

model = Autoencoder(kernel_size=5)

model = model.cuda()
trainset = TrainDataset()
trainloader = data.DataLoader(trainset, batch_size=1)

testset = TestDataset()
testloader = data.DataLoader(testset, batch_size=1)

# Train the model
print("---TRAIN for 3 batches---")
print(f"Input and expected output size: {trainloader.dataset[0].shape}")

for i, input in enumerate(trainloader):
    if i == 3: 
        break
    input = input.cuda()
    output = model(input)
    print(f"Output size: {output.shape}")

print("---TESTING ON CPU----")
print(f"Input and expected output size: {testloader.dataset[0].shape}")
# Test the model on cpu
print("")
model.cpu()
for i, input in enumerate(testloader):
    if i == 1: 
        break
    output = model(input)
    print(f"Output size: {output.shape}")

print("---TESTING ON GPU (PROBLEMATIC)----")
print(f"Input and expected output size: {testloader.dataset[0].shape}")
model.cuda()
for i, input in enumerate(testloader):
    if i == 1: 
        break
    input = input.cuda()
    output = model(input)
    print(f"Output size: {output.shape}")