rapidsai_xavier_install_script_v1

#!/bin/bash

# Global Arguments
BUILD_ROOT=~/Development
CMAKE_ROOT=$BUILD_ROOT/cmake
LLVM_ROOT=$BUILD_ROOT/llvm
LIBCUDF_BUILD_ROOT=$BUILD_ROOT/cudf/cpp/build
CUDF_BUILD_ROOT=$BUILD_ROOT/cudf/python/cudf

echo "======== RapidsAI Xavier Installation Script ========"

# Prepares the device for the installation
prepare_device() {
	echo "Setting Xavier device power management to 'MAXN' for faster compilation"
	nvpmodel -m 0
}

# Installs system dependencies
install_system_deps() { 
	apt-get update -y
	apt-get install -y libssl-dev libboost-all-dev python3.7 python3-pip
	update-alternatives --install /usr/bin/python python /usr/bin/python3 10
	mkdir -p $CMAKE_ROOT
	cd $CMAKE_ROOT
	wget https://nvidia-xavier.s3.amazonaws.com/cmake-3.16.2-Linux-aarch64.tar.gz
	# wget https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz # source code for this binary build
	tar -xzvf ./cmake-3.16.2-Linux-aarch64.tar.gz
	cd cmake-3.16.2-Linux-aarch64
	echo "export PATH=$(pwd)/bin:$PATH" >> /etc/profile
	export PATH="$(pwd)/bin:$PATH"
	mkdir -p $LLVM_ROOT
	cd $LLVM_ROOT
	# wget http://releases.llvm.org/7.0.1/llvm-7.0.1.src.tar.xz # source code for this binary build
	wget https://nvidia-xavier.s3.amazonaws.com/LLVM-7.0.1-Linux.tar.gz
	tar -xvf ./LLVM-7.0.1-Linux.tar.gz
	cd LLVM-7.0.1-Linux
	echo "export LLVM_CONFIG=$(pwd)/bin/llvm-config" >> /etc/profile
	export LLVM_CONFIG="$(pwd)/bin/llvm-config"
	echo $LLVM_CONFIG
}

install_python_deps() {
	#For the life of me I can't figure out why pip is not liking all the 
	# installs on one line but whatever will do them on individual lines...
	pip3 install wheel
	pip3 install cmake_setuptools
	pip3 install llvmlite 
	pip3 install cython
	apt remove python3-numpy
	pip3 install numpy
	pip3 install numba==0.46.0
	pip3 install fastavro
	pip3 install fsspec
	apt remove python3-pandas
	pip3 install 'pandas>=0.25,<0.26'
	pip3 install cupy
}

# Setup build environment
prepare_build_env() {
	echo "export CUDA_HOME=/usr/local/cuda-10.2" >> /etc/profile
	export CUDA_HOME=/usr/local/cuda-10.2
	echo "export CUDACXX=/usr/local/cuda-10.2/bin/nvcc" >> /etc/profile
	export CUDACXX=/usr/local/cuda-10.2/bin/nvcc
	echo "export ARROW_HOME=$BUILD_ROOT/cudf/cpp/build/arrow/install" >> /etc/profile
	export ARROW_HOME=$BUILD_ROOT/cudf/cpp/build/arrow/install
	echo "export DLPACK_ROOT=$BUILD_ROOT/dlpack" >> /etc/profile
	export DLPACK_ROOT=$BUILD_ROOT/dlpack
	echo "export DLPACK_INCLUDE=$BUILD_ROOT/dlpack/include" >> /etc/profile
	export DLPACK_INCLUDE=$BUILD_ROOT/dlpack/include

	mkdir -p $BUILD_ROOT

	# Clone cudf codebase
	# git clone https://github.com/rapidsai/cudf.git $BUILD_ROOT/cudf

	# Clone my branch until needed changes are merged
	git clone https://github.com/jdye64/cudf.git $BUILD_ROOT/cudf
	cd $BUILD_ROOT/cudf
	git checkout cudf_xavier
	git submodule update --init --remote --recursive

	# Clone dlpack dependency
	git clone https://github.com/dmlc/dlpack.git $BUILD_ROOT/dlpack
	export DLPACK_HOME=$BUILD_ROOT/dlpack

	# Clone and build rmm dependency codebase
	git clone https://github.com/rapidsai/rmm $BUILD_ROOT/rmm
	cd $BUILD_ROOT/rmm
	git submodule update --init --remote --recursive

	# Build RMM
	./build.sh

	mkdir -p $LIBCUDF_BUILD_ROOT
}

# Builds and installs RapidsAI libcudf and cudf
build_cudf() {
	cd $LIBCUDF_BUILD_ROOT
	cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_ALL_GPU_ARCH=0 -DBUILD_NVTX=OFF -DBUILD_TESTS=OFF -DBUILD_LEGACY_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDISABLE_DEPRECATION_WARNING=ON -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_CXX11_ABI=ON ..
	make -j4
	make install

	# Set the Arrow install location for pip can install pyarrow
	export ARROW_HOME=$LIBCUDF_BUILD_ROOT/arrow/install
	pip3 install pyarrow==0.15.1

	cd $CUDF_BUILD_ROOT
	python setup.py build_ext --inplace
	python setup.py install --single-version-externally-managed --record=record.txt
}

# Tests out the installation
test_install() {
	python -c "import cudf"
}

GLOBALTIME=$(date +%s)

STARTTIME=$(date +%s)
echo "Preparing device for build"
prepare_device
ENDTIME=$(date +%s)
echo "Device preparation took: $(($ENDTIME - $STARTTIME)) seconds to complete"

STARTTIME=$(date +%s)
echo "Install System Dependencies"
install_system_deps
ENDTIME=$(date +%s)
echo "Installing System Dependencies took: $(($ENDTIME - $STARTTIME)) seconds to complete"

STARTTIME=$(date +%s)
echo "Install Python Dependencies"
install_python_deps
ENDTIME=$(date +%s)
echo "Installing Python Dependencies took: $(($ENDTIME - $STARTTIME)) seconds to complete"

STARTTIME=$(date +%s)
echo "Preparing build environment"
prepare_build_env
ENDTIME=$(date +%s)
echo "Preparing build environment took: $(($ENDTIME - $STARTTIME)) seconds to complete"

STARTTIME=$(date +%s)
echo "Building cuDF"
build_cudf
ENDTIME=$(date +%s)
echo "Building cuDF took: $(($ENDTIME - $STARTTIME)) seconds to complete"

ENDTIME=$(date +%s)
echo "RapidsAI CUDF Install took: $(($ENDTIME - $GLOBALTIME)) seconds to complete"

# Testing the installation
test_install