logs.txt

root@fe83e7c706f1:/home/ICON/apps# python infer.py -cfg ../configs/icon-filter.yaml -gpu 0 -in_dir ../examples -out_dir ../results

Using /root/.cache/torch_extensions as PyTorch extensions root...
Creating extension directory /root/.cache/torch_extensions/voxelize_cuda...
Detected CUDA files, patching ldflags
Emitting ninja build file /root/.cache/torch_extensions/voxelize_cuda/build.ninja...
Building extension module voxelize_cuda...
Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
[1/3] c++ -MMD -MF voxelize_cuda.o.d -DTORCH_EXTENSION_NAME=voxelize_cuda -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /opt/conda/lib/python3.8/site-packages/torch/include -isystem /opt/conda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.8/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.8/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /home/ICON/lib/neural_voxelization_layer/cuda/voxelize_cuda.cpp -o voxelize_cuda.o
[2/3] /usr/local/cuda/bin/nvcc --generate-dependencies-with-compile --dependency-output voxelize_cuda.cuda.o.d -DTORCH_EXTENSION_NAME=voxelize_cuda -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /opt/conda/lib/python3.8/site-packages/torch/include -isystem /opt/conda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.8/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.8/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_80,code=compute_80 -gencode=arch=compute_80,code=sm_80 --compiler-options '-fPIC' -std=c++14 -c /home/ICON/lib/neural_voxelization_layer/cuda/voxelize_cuda.cu -o voxelize_cuda.cuda.o
/home/ICON/lib/neural_voxelization_layer/cuda/voxelize_cuda.cu(121): warning: variable "ti" was declared but never referenced

[3/3] c++ voxelize_cuda.o voxelize_cuda.cuda.o -shared -L/usr/local/cuda/targets/x86_64-linux/lib -L/opt/conda/lib/python3.8/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda_cu -ltorch_cuda_cpp -ltorch -ltorch_python -L/usr/local/cuda/lib64 -lcudart -o voxelize_cuda.so
Loading extension module voxelize_cuda...
OMP: Info #271: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.
normal environment...
ICON:
[w/ Global Image Encoder]: True
[Image Features used by MLP]: ['normal_F', 'normal_B']
[Geometry Features used by MLP]: ['sdf', 'norm', 'vis', 'cmap']
[Dim of Image Features (local)]: 6
[Dim of Geometry Features (ICON)]: 7
[Dim of MLP's first layer]: 13

initialize network with xavier
initialize network with xavier
Resume MLP weights from ../data/ckpt/icon-filter.ckpt
Resume normal model from ../data/ckpt/normal.ckpt
Downloading: "https://github.com/NVIDIA/DeepLearningExamples/archive/torchhub.zip" to /root/.cache/torch/hub/torchhub.zip
Using cache found in /root/.cache/torch/hub/NVIDIA_DeepLearningExamples_torchhub
Dataset Size: 9
Body Fitting = 0.182: 100%|███████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:11<00:00,  8.34it/s]
22097467bffc92d4a5c4246f7d4edb75:   0%|                                                                                               | 0/9 [00:16<?, ?it/s]
Traceback (most recent call last):                                                                                                   | 0/10 [00:00<?, ?it/s]
  File "infer.py", line 301, in <module>
    verts_pr, faces_pr, _ = model.test_single(in_tensor)
  File "/home/ICON/apps/ICON.py", line 738, in test_single
    sdf = self.reconEngine(opt=self.cfg,
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ICON/apps/../lib/common/seg3d_lossless.py", line 148, in forward
    return self._forward_faster(**kwargs)
  File "/home/ICON/apps/../lib/common/seg3d_lossless.py", line 170, in _forward_faster
    occupancys = self.batch_eval(coords, **kwargs)
  File "/home/ICON/apps/../lib/common/seg3d_lossless.py", line 139, in batch_eval
    occupancys = self.query_func(**kwargs, points=coords2D)
  File "/home/ICON/apps/../lib/common/train_util.py", line 338, in query_func
    preds = netG.query(features=features,
  File "/home/ICON/apps/../lib/net/HGPIFuNet.py", line 285, in query
    smpl_sdf, smpl_norm, smpl_cmap, smpl_ind = cal_sdf_batch(
  File "/home/ICON/apps/../lib/dataset/mesh_util.py", line 231, in cal_sdf_batch
    residues, normals, pts_cmap, pts_ind = func(
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/bvh_distance_queries/mesh_distance.py", line 79, in forward
    output = self.search_tree(triangles, points)
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/bvh_distance_queries/bvh_search_tree.py", line 109, in forward
    output = BVHFunction.apply(
  File "/opt/conda/lib/python3.8/site-packages/bvh_distance_queries/bvh_search_tree.py", line 42, in forward
    outputs = bvh_distance_queries_cuda.distance_queries(
RuntimeError: after reduction step 1: cudaErrorInvalidDevice: invalid device ordinal
  0%|                                                                                                                                | 0/10 [00:01<?, ?it/s]