train_trex.sh

#!/bin/sh
#SBATCH --error=output/nsvf_trex.error.%j  # STDOUT output is written in slurm.out.JOBID
#SBATCH --output=output/nsvf_trex.out.%j # STDOUT error is written in slurm.err.JOBID
#SBATCH --job-name=trex_nsvf       # Job name
#SBATCH --mem=32GB                  # Memory request for this job
#SBATCH --nodes=1                   # The number of nodes
#SBATCH --partition=gpu-cluster
#SBATCH --account=vision
#SBATCH --time=48:0:0                # Runing time 2 days
#SBATCH --gpus=1                    # A number of GPUs
module load Anaconda3
module load CUDA/10.2
module load cuDNN/7
module load OpenMPI/3.1.4-GCC-8.3.0

. /ist/apps/modules/software/Anaconda3/5.3.0/etc/profile.d/conda.sh
conda activate venv_nsvf

DATA="trex_nsvf"
RES="756x1008"
ARCH="nsvf_base"
SUFFIX="v1"
DATASET=./data/${DATA}
SAVE=./runs/$DATA
MODEL=$ARCH$SUFFIX
mkdir -p $SAVE/$MODEL


# start training locally
python train.py ${DATASET} \
        --user-dir fairnr \
        --task single_object_rendering \
        --train-views "0..47" \
        --view-resolution $RES \
        --max-sentences 1 \
        --view-per-batch 4 \
        --pixel-per-view 2048 \
        --no-preload \
        --sampling-on-mask 1.0 --no-sampling-at-reader \
        --valid-view-resolution $RES \
        --valid-views "47..55" \
        --valid-view-per-batch 1 \
        --transparent-background "0.0,0.0,0.0" \
        --background-stop-gradient \
        --arch $ARCH \
        --initial-boundingbox ${DATASET}/bbox.txt \
        --raymarching-stepsize-ratio 0.125 \
        --use-octree \
        --discrete-regularization \
        --color-weight 128.0 \
        --alpha-weight 1.0 \
        --optimizer "adam" \
        --adam-betas "(0.9, 0.999)" \
        --lr-scheduler "polynomial_decay" \
        --total-num-update 150000 \
        --lr 0.001 \
        --clip-norm 0.0 \
        --criterion "srn_loss" \
        --num-workers 8 \
        --seed 2 \
        --save-interval-updates 500 --max-update 100000 \
        --virtual-epoch-steps 5000 --save-interval 1 \
        --half-voxel-size-at  "5000,25000" \
        --reduce-step-size-at "5000,25000" \
        --pruning-every-steps 2500 \
        --keep-interval-updates 5 \
        --log-format simple --log-interval 1 \
        --tensorboard-logdir ${SAVE}/tensorboard/${MODEL} \
        --save-dir ${SAVE}/${MODEL}