Reproduce Regression

Dockerfile

FROM kellens/build.ubuntu_gpu_tensorrt:latest

WORKDIR /work

RUN wget https://gist.githubusercontent.com/KellenSunderland/686522830475dfc7073b5d7a97e89d24/raw/a0b12e63b5fbf51f4c2794a9a8dae22a2ac8cab1/run_model.py && \
    wget https://raw.githubusercontent.com/apache/incubator-mxnet/master/example/fcn-xs/symbol_fcnxs.py

RUN git clone --recursive https://github.com/apache/incubator-mxnet.git --branch v1.4.x mxnet-v1.4.0-cmake && \
    mkdir -p /work/mxnet-v1.4.0-cmake/lib/ && \
    git clone --recursive https://github.com/apache/incubator-mxnet.git --branch v1.4.x mxnet-v1.4.0-make && \
    mkdir -p /work/mxnet-v1.4.0-make/lib/ && \
    git clone --recursive https://github.com/KellenSunderland/incubator-mxnet.git --branch nvtx_initial_merge mxnet-v1.4.0-cmake-profiler && \
    mkdir -p /work/mxnet-v1.4.0-cmake-profiler/lib/  && \
    git clone --recursive https://github.com/apache/incubator-mxnet.git --branch v1.3.x mxnet-v1.3.0-cmake && \
    mkdir -p /work/mxnet-v1.3.0-cmake/lib/

WORKDIR /work/mxnet-v1.4.0-cmake/lib

RUN cmake .. -DUSE_LAPACK=OFF \
             -DUSE_MKL_IF_AVAILABLE=OFF \
             -DUSE_OPERATOR_TUNING=OFF \
             -DUSE_CPP_PACKAGE=ON \
             -DUSE_OPENCV=ON \
             -DBUILD_CPP_EXAMPLES=OFF \
             -DBLAS=open \
             -DUSE_CUDNN=ON \
             -DCUDA_ARCH_NAME=Manual \
             -DCUDA_ARCH_BIN=70 \
             -DENABLE_CUDA_RTC=OFF \
             -DUSE_F16C=OFF && \
             make -j $(nproc)

WORKDIR /work/mxnet-v1.4.0-cmake-profiler/lib

RUN cmake .. -DUSE_LAPACK=OFF \
             -DUSE_MKL_IF_AVAILABLE=OFF \
             -DUSE_OPERATOR_TUNING=OFF \
             -DUSE_CPP_PACKAGE=ON \
             -DUSE_OPENCV=ON \
             -DBUILD_CPP_EXAMPLES=OFF \
             -DBLAS=open \
             -DUSE_CUDNN=ON \
             -DCUDA_ARCH_NAME=Manual \
             -DCUDA_ARCH_BIN=70 \
             -DENABLE_CUDA_RTC=OFF \
             -DUSE_F16C=OFF && \
             make -j $(nproc)

WORKDIR /work/mxnet-v1.3.0-cmake/lib

RUN cmake .. -DUSE_LAPACK=OFF \
             -DUSE_MKL_IF_AVAILABLE=OFF \
             -DUSE_OPERATOR_TUNING=OFF \
             -DUSE_CPP_PACKAGE=ON \
             -DUSE_OPENCV=ON \
             -DBUILD_CPP_EXAMPLES=OFF \
             -DBLAS=open \
             -DUSE_OPENMP=OFF \
             -DUSE_CUDNN=ON \
             -DCUDA_ARCH_NAME=Manual \
             -DCUDA_ARCH_BIN=70 \
             -DENABLE_CUDA_RTC=OFF \
             -DUSE_OLDCMAKECUDA=ON \
             -DUSE_F16C=OFF && \
             make -j $(nproc)

WORKDIR /work/mxnet-v1.4.0-make/

RUN wget https://gist.githubusercontent.com/KellenSunderland/4cd6c6368295c33ecb43ffa2773eb234/raw/9483aadc405572ed3ce8240ddc12535c179a2868/config.mk

WORKDIR /work/mxnet-v1.4.0-make/make

RUN rm config.mk && \
    wget https://gist.githubusercontent.com/KellenSunderland/4cd6c6368295c33ecb43ffa2773eb234/raw/9483aadc405572ed3ce8240ddc12535c179a2868/config.mk

WORKDIR /work/mxnet-v1.4.0-make/

RUN make -j $(nproc)

WORKDIR /work

RUN git clone --recursive https://github.com/apache/incubator-mxnet.git --branch v1.2.0 mxnet-v1.2.0-cmake && \
    mkdir -p /work/mxnet-v1.2.0-cmake/lib/

WORKDIR /work/mxnet-v1.2.0-cmake/lib

RUN cmake .. -DUSE_LAPACK=OFF \
             -DUSE_MKL_IF_AVAILABLE=OFF \
             -DUSE_OPERATOR_TUNING=OFF \
             -DUSE_CPP_PACKAGE=ON \
             -DUSE_OPENCV=ON \
             -DBUILD_CPP_EXAMPLES=OFF \
             -DBLAS=open \
             -DUSE_OPENMP=OFF \
             -DUSE_CUDNN=ON \
             -DCUDA_ARCH_NAME=Manual \
             -DCUDA_ARCH_BIN=70 \
             -DENABLE_CUDA_RTC=OFF \
             -DUSE_OLDCMAKECUDA=ON \
             -DUSE_F16C=OFF && \
             make -j $(nproc)

WORKDIR /work/