domarps/caffe-setup.md

## caffe-setup.md

      
    Raw
  

              caffe-setup.md
            
          
    sketchstyle-barebones

Steps to install caffe with extra layer


Install NCCL

git clone https://github.com/NVIDIA/nccl.git /home/ubuntu/nccl --recursive
make CUDA_HOME=/usr/local/cuda -j4
make install


Install caffe; Copy Makefile.config file and caffenorm files as follows

git clone https://github.com/BVLC/caffe.git /home/ubuntu/caffe --recursive
cp caffenorm/*.hpp /home/ubuntu/caffe/include/caffe/layers
cp caffenorm/*.cpp /home/ubuntu/caffe/src/caffe/layers
cp caffenorm/*.cu /home/ubuntu/caffe/src/caffe/layers
apt-get install -y libprotobuf-dev libleveldb-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler libhdf5-dev libatlas-base-dev
apt-get install -y --no-install-recommends libboost-all-dev
apt-get install -y libgflags-dev libgoogle-glog-dev liblmdb-dev mlocate
cp Makefile.config /home/ubuntu/caffe/Makefile.config
RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial.so.10 /usr/lib/x86_64-linux-gnu/libhdf5.so
RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10 /usr/lib/x86_64-linux-gnu/libhdf5_hl.so


Install opencv3.1.0

cd /tmp \
   && git clone https://github.com/Itseez/opencv.git \
   && cd opencv \
   && git checkout 3.1.0 \
   && mkdir build \
   && cd build \
   && cmake -D CMAKE_BUILD_TYPE=RELEASE \
       -D CMAKE_INSTALL_PREFIX=/usr/local \
       -D WITH_LAPACK=OFF \
       -D WITH_CUDA=OFF \
       -D ENABLE_AVX=ON \
       -D WITH_OPENGL=OFF \
       -D WITH_OPENCL=OFF \
       -D WITH_IPP=OFF \
       -D WITH_TBB=OFF \
       -D WITH_EIGEN=ON \
       -D WITH_V4L=ON \
       -D BUILD_EXAMPLES=OFF \
       -D BUILD_TESTS=OFF \
       -D BUILD_opencv_java=OFF \
       -D BUILD_opencv_python3=ON \
       -D PYTHON_EXECUTABLE=$(which python3) \
       -D PYTHON_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
       -D PYTHON_LIBRARY=$(python3 -c "from distutils.sysconfig import get_config_var;from os.path import dirname,join ; print(join(dirname(get_config_var('LIBPC')),get_config_var('LDLIBRARY')))") \
       -D PYTHON_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
       -D PYTHON3_NUMPY_INCLUDE_DIRS=$(python3 -c "import numpy; print(numpy.get_include())") \
       .. \
   && make -j4 \
   && make install \
   && ldconfig \
   && cd /tmp && rm -rf /tmp/opencv*


Uncomment USE_PKG_CONFIG := 1 in the Makefile.config (Ref)


Run the following commands:


cp make pycaffe -j4
make distribute


Output of make distribute

AR -o .build_release/lib/libcaffe.a
CXX tools/extract_features.cpp
CXX/LD -o .build_release/tools/extract_features.bin
CXX tools/upgrade_net_proto_text.cpp
CXX/LD -o .build_release/tools/upgrade_net_proto_text.bin
CXX tools/convert_imageset.cpp
CXX/LD -o .build_release/tools/convert_imageset.bin
CXX tools/compute_image_mean.cpp
CXX/LD -o .build_release/tools/compute_image_mean.bin
CXX tools/caffe.cpp
CXX/LD -o .build_release/tools/caffe.bin
CXX tools/upgrade_net_proto_binary.cpp
CXX/LD -o .build_release/tools/upgrade_net_proto_binary.bin
CXX tools/upgrade_solver_proto_text.cpp
CXX/LD -o .build_release/tools/upgrade_solver_proto_text.bin
CXX examples/mnist/convert_mnist_data.cpp
CXX/LD -o .build_release/examples/mnist/convert_mnist_data.bin
CXX examples/cpp_classification/classification.cpp
CXX/LD -o .build_release/examples/cpp_classification/classification.bin
CXX examples/cifar10/convert_cifar_data.cpp
CXX/LD -o .build_release/examples/cifar10/convert_cifar_data.bin
CXX examples/siamese/convert_mnist_siamese_data.cpp
CXX/LD -o .build_release/examples/siamese/convert_mnist_siamese_data.bin
# add proto
cp -r src/caffe/proto distribute/
# add include
cp -r include distribute/
mkdir -p distribute/include/caffe/proto
cp .build_release/src/caffe/proto/caffe.pb.h distribute/include/caffe/proto
# add tool and example binaries
cp .build_release/tools/extract_features.bin .build_release/tools/upgrade_net_proto_text.bin .build_release/tools/convert_imageset.bin .build_release/tools/compute_image_mean.bin .build_release/tools/caffe.bin .build_release/tools/upgrade_net_proto_binary.bin .build_release/tools/upgrade_solver_proto_text.bin distribute/bin
cp .build_release/examples/mnist/convert_mnist_data.bin .build_release/examples/cpp_classification/classification.bin .build_release/examples/cifar10/convert_cifar_data.bin .build_release/examples/siamese/convert_mnist_siamese_data.bin distribute/bin
# add libraries
cp .build_release/lib/libcaffe.a distribute/lib
install -m 644 .build_release/lib/libcaffe.so.1.0.0 distribute/lib
cd distribute/lib; rm -f libcaffe.so;   ln -s libcaffe.so.1.0.0 libcaffe.so
# add python - it's not the standard way, indeed...
cp -r python distribute/


Update PYTHONPATH

# in caffe root dir
make all
make pycaffe
make distribute
# make dir for custom python modules, install caffe
mkdir ~/python
mv distribute/python/caffe ~/python
# set PYTHONPATH (this should go in your .bashrc or whatever
PYTHONPATH=${HOME}/python:$PYTHONPATH


Update LD_LIBRARY_PATH

LD_LIBRARY_PATH=/home/ubuntu/caffe/distribute/lib/:$LD_LIBRARY_PATH

Test caffe installation

>>> import caffe
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/ubuntu/python/caffe/__init__.py", line 1, in <module>
    from .pycaffe import Net, SGDSolver, NesterovSolver, AdaGradSolver, RMSPropSolver, AdaDeltaSolver, AdamSolver, NCCL, Timer
  File "/home/ubuntu/python/caffe/pycaffe.py", line 13, in <module>
    from ._caffe import Net, SGDSolver, NesterovSolver, AdaGradSolver, \
ImportError: dynamic module does not define module export function (PyInit__caffe)