Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
# This is script version 3.0 - HPC & PyClaw in Mac OSX
# ------------------------------------------------------------------------------ #
# This version uses github and bitbucket sources
# ------------------------------------------------------------------------------ #
# ------------------------------------------------------------------------------ #
#!/bin/bash -x
echo -e "\n**************************************************************"
echo "Build a HPC system in MacOSX"
echo "**************************************************************"
echo "hpc_prep - set up build environment"
hpc_prep() {
# set working directories
echo "hpc_preparation"
echo "setting up directories"
projdir=/sandbox
echo "projdir=${projdir}"
sandbox=${projdir}
echo "sandbox=${sandbox}"
builddir=${sandbox}/opt/share
echo "builddir=${builddir}"
srcdir=${builddir}/sources
echo "srcdir=${srcdir}"
logdir=${builddir}/logs
echo "logdir=${logdir}"
appsdir=${sandbox}/apps
echo "appsdir=${appsdir}"
simdir=${sandbox}/simdesk
echo "simdir=${simdir}"
devdir=${sandbox}/dev
echo "devdir=${devdir}"
# build directories
mkdir -p ${builddir} ${srcdir} ${logdir} ${appsdir} ${simdir} ${devdir}
# identify user profiles
profileb=$HOME/.bash_profile
# packages versions
mpich_branch=master
anaconda_ver=2.5.0
psutil_ver=4.1.0
mprofiler_ver=0.41
cython_branch=master
mpi4py_branch=master
petsc_branch=maint
petsc_subdir=maint
fftw_ver=3.3.4
fftw_branch=master
sowing_ver=1.1.18-p1
petsc4py_branch=maint
clawpack_branch=master
ofoam_branch=master
ofoam_version=2.4.x
ofoam_third_party=2.4.0
paraview_branch=release
cmake_branch=master
qt_ver=4.8
qt_release=4.8.6
glibc_release=master
sublime_version=2.0.2
r_release=3
r_version=3.2.4
n_cores=8
# fblaslapack_ver=3.4.2
# f2cblaslapack_ver=3.4.2.q
# scalapack_ver=2.0.2
# hdf5_ver=1.8.13
has_submodules=("clawpack paraview")
files=( "${mpich_branch}" \
"Anaconda2-${anaconda_ver}-MacOSX-x86_64.sh" \
"psutil-${psutil_ver}.tar.gz" \
"memory_profiler-${mprofiler_ver}.tar.gz" \
"${cython_branch}" \
"${mpi4py_branch}" \
"fftw-${fftw_ver}.tar.gz" \
"${fftw_branch}" \
"${petsc_branch}" \
"${petsc4py_branch}" \
"${clawpack_branch}" \
"${ofoam_branch}" \
"ofoam_third_party_${ofoam_third_party}" \
"${paraview_branch}" \
"${cmake_branch}" \
"qt-everywhere-opensource-src-${qt_release}.tar.gz" \
"${glibc_release}" \
"R_${r_version}.tar.gz" \
)
urls=( "http://git.mpich.org/mpich.git mpich" \
"http://repo.continuum.io/archive/Anaconda2-${anaconda_ver}-MacOSX-x86_64.sh" \
"https://pypi.python.org/packages/source/p/psutil/psutil-${psutil_ver}.tar.gz" \
"https://pypi.python.org/packages/source/m/memory_profiler/memory_profiler-${mprofiler_ver}.tar.gz" \
"git@github.com:cython/cython.git cython" \
"git@bitbucket.org:mpi4py/mpi4py.git mpi4py" \
"http://www.fftw.org/fftw-${fftw_ver}.tar.gz" \
"git@github.com:FFTW/fftw3.git fftw" \
"git@bitbucket.org:petsc/petsc.git petsc" \
"git@bitbucket.org:petsc/petsc4py.git petsc4py" \
"git@github.com:clawpack/clawpack.git ${simdir}/clawpack" \
"git@github.com:OpenFOAM/OpenFOAM-${ofoam_version}.git ofoam" \
"http://downloads.sourceforge.net/foam/ThirdParty-${ofoam_third_party}.tgz" \
"https://gitlab.kitware.com/paraview/paraview.git paraview" \
"https://cmake.org/cmake.git cmake" \
"http://download.qt.io/archive/qt/${qt_ver}/${qt_release}/qt-everywhere-opensource-src-${qt_release}.tar.gz" \
"git://sourceware.org/git/glibc.git glibc" \
"https://cran.r-project.org/src/base/R-${r_release}/R-${r_version}.tar.gz" \
)
#disable_threads="yes"
#if [ disable_threads == "yes" ] ; then
#threads_flag="--without-threads"
# echo "Python and extension modules will be built without thread support"
#else
threads_flag="--with-threads"
# echo "Python and extension modules will be built with thread support"
#fi
pic_flag="-fpic"
source ${profileb}
}
_listcontains() {
for word in $1; do
if [[ ${2} == *${word}* ]]; then
echo "${2} and word ${word}"
return 0
fi
done
return 1
}
_git_init_submodule(){
branch_name=$1
echo -e "initializing submodules"
git submodule init
git submodule update
git submodule foreach git checkout ${branch_name}
git submodule foreach git pull
}
_git_post(){
repo_dir=$1
branch_name=$2
cd ${repo_dir}
git fetch --tags
latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`)
echo "updating repo ${repo_dir}"
if _listcontains "${has_submodules}" ${repo_dir}; then
# echo Y; else echo N; fi
# if [[ ${has_submodules} =~ ${repo_dir} ]]; then
echo " going into submodules for ${repo_dir}"
_git_init_submodule ${branch_name}
elif [[ ${repo_dir} == *"glibc"* ]]; then
git checkout --track -b release `git branch --remotes --list origin/release* | tail -1`
git pull
else
git checkout ${branch_name}
git pull
fi
}
_get_src(){
src_file=$1
src_url=$2
cd ${srcdir}
if [[ ${src_url} == *".git"* ]]; then
repo_dir=$3
if [ ! -d ${repo_dir} ]; then
echo "#### cloning repo ${src_url} into ${repo_dir} #####"
echo -e ">> git clone ${src_url} ${repo_dir}"
git clone ${src_url} ${repo_dir}
fi
echo "pull and update repo ${repo_dir}"
_git_post ${repo_dir} ${src_file}
else
echo -e "#### downloading ${src_file} to ${srcdir} #####"
if [ ! -f ${srcdir}/${src_file} ]; then
echo -e ">> wget -P ${srcdir} -O ${src_file} ${src_url}"
wget -P ${srcdir} -O ${src_file} ${src_url}
else
echo -e 'src file is present, skipping\n'
fi
fi
}
_extract_src() {
src_file=$1
cd ${srcdir}
tar -zxvf ${srcdir}/${src_file}
}
_add_to_profile() {
echo "adding to profile"
who_added=$1
line=$2
profile=$3
echo -e "\n# added by ${who_added}\n${line}\n" >> ${profile}
}
echo "hpc_get_sources - get all sources"
hpc_get_sources() {
hpc_prep
echo "getting sources"
cd ${srcdir}
nfiles=${#files[@]}
nfiles=$((nfiles-1))
echo " There are $nfiles repositories/files to clone"
for (( i=0; i<${nfiles}+1; i++ ));
do
echo $i "/" ${nfiles} "getting: " ${files[$i]} " from " ${urls[$i]}
_get_src ${files[$i]} ${urls[$i]}
done
cd ${projdir}
}
echo "hpc_build_mpich - install mpich2 "
hpc_build_mpich(){
hpc_prep
cd ${srcdir}/mpich
echo "installing mpich to the system"
if [ ! -f configure ]; then
echo "first running autogen.sh to compile installation files"
./autogen.sh \
2>&1 | tee ${logdir}/mpich-${mpich_branch}_autogen.log
fi
echo "configure"
./configure \
CFLAGS=-fPIC FFLAGS=-fPIC \
--with-threads \
--enable-shared --enable-sharedlibs=gcc \
--enable-fast=all,03 \
-prefix=${builddir}/mpich/${mpich_branch} \
--disable-multi-aliases \
2>&1 | tee ${logdir}/mpich-${mpich_branch}_configure.log
echo "make build"
make \
2>&1 | tee ${logdir}/mpich-${mpich_branch}_make.log
echo "make install"
make install \
2>&1 | tee ${logdir}/mpich-${mpich_branch}_install.log
mpich_dir=${builddir}/mpich/${mpich_branch}/bin
export PATH=${mpich_dir}:$PATH
_add_to_profile "mpich-${mpich_branch}" "export PATH=\"${mpich_dir}:\$PATH\" \n" ${profileb}
}
echo "hpc_build_anaconda - install anaconda ${anaconda_ver}"
hpc_build_anaconda() {
echo "hpc_build_anaconda"
hpc_prep
cd ${srcdir}
bash ${srcdir}/Anaconda2-${anaconda_ver}-MacOSX-x86_64.sh \
2>&1 | tee ${logdir}/Anaconda-${anaconda_ver}_install.log
}
echo "hpc_build_psutil - install psutils ${psutil_ver}"
hpc_build_psutil() {
echo "hpc_build_psutil"
hpc_prep
cd ${srcdir}
tar -zxvf ${srcdir}/psutil-${psutil_ver}.tar.gz
cd psutil-${psutil_ver}
python setup.py -v build \
2>&1 | tee ${logdir}/psutil-${psutil_ver}_build.log
python setup.py -v install \
2>&1 | tee ${logdir}/psutil-${psutil_ver}_install.log
}
echo "hpc_build_profiler - install memory profiler ${mprofiler_ver}"
hpc_build_profiler() {
echo "hpc_build_profiler"
hpc_prep
cd ${srcdir}
tar -zxvf ${srcdir}/memory_profiler-${mprofiler_ver}.tar.gz
cd memory_profiler-${mprofiler_ver}
python setup.py -v build \
2>&1 | tee ${logdir}/memory_profiler-${mprofiler_ver}_build.log
python setup.py -v install \
2>&1 | tee ${logdir}/memory_profiler-${mprofiler_ver}_install.log
}
echo "hpc_build_cython - install cython"
hpc_build_cython() {
echo "hpc_build_cython"
hpc_prep
cd ${srcdir}/cython
python setup.py -v build_ext install \
2>&1 | tee ${logdir}/cython_install.log
}
echo "hpc_build_mpi4py - install mpi4py ${mpi4py_branch}"
hpc_build_mpi4py() {
echo "hpc_build_mpi4py"
hpc_prep
cd ${srcdir}/mpi4py
python setup.py -v build_src build_ext install \
2>&1 | tee ${logdir}/mpi4py-${mpi4py_branch}_install.log
python setup.py test \
2>&1 | tee ${logdir}/mpi4py-${mpi4py_branch}_test.log
}
echo "hpc_build_petsc - install petsc"
hpc_build_petsc() {
echo "hpc_build_petsc"
hpc_prep
cd ${srcdir}/petsc
echo "configuration"
unset PETSC_DIR
./configure \
--known-mpi-shared-libraries=1 \
--with-shared-libraries=1 \
--with-64-bit-indices=1 \
--download-fblaslapack \
--download-scalapack \
--download-sowing \
--download-hdf5 \
-CFLAGS=-g -CXXFLAGS=-g \
-FFLAGS=-g --with-debugging=1 \
-COPTFLAGS=-g \
-FOPTFLAGS=-g \
LIBS=-dynamic \
PETSC_ARCH=arch-osx-so \
--prefix=${builddir}/petsc/${petsc_branch}/osx \
2>&1 | tee ${logdir}/petsc-${petsc_branch}_configure.log
make PETSC_DIR=${srcdir}/petsc PETSC_ARCH=arch-osx-so all \
2>&1 | tee ${logdir}/petsc-${petsc_branch}_make.log
make PETSC_DIR=${srcdir}/petsc PETSC_ARCH=arch-osx-so install \
2>&1 | tee ${logdir}/petsc-${petsc_branch}_make_install.log
make PETSC_DIR=${builddir}/petsc/${petsc_branch}/osx PETSC_ARCH="" test \
2>&1 | tee ${logdir}/petsc-${petsc_branch}_make_test.log
make PETSC_DIR=${builddir}/petsc/${petsc_branch}/osx PETSC_ARCH= streams NPMAX=${n_cores} \
2>&1 | tee ${logdir}/petsc-${petsc_branch}_make_profile.log
echo -e "\n# added by PETSc install, version ${petsc_branch}" >> ${profileb}
echo -e "export PETSC_DIR=${builddir}/petsc/${petsc_branch}/osx \n" >> ${profileb}
echo -e "export PETSC_ARCH=arch-osx-so \n" >> ${profileb}
export PETSC_DIR=${builddir}/petsc/${petsc_branch}/osx
}
echo "hpc_build_petsc4py - install petsc4py"
hpc_build_petsc4py() {
echo "hpc_build_petsc4py"
hpc_prep
cd ${srcdir}/petsc4py
export PETSC_DIR=${builddir}/petsc/${petsc_branch}/osx
python setup.py -v build_src build_ext install \
2>&1 | tee ${logdir}/petsc4py-${petsc4py_branch}_install.log
python setup.py test \
2>&1 | tee ${logdir}/petsc4py-${petsc4py_branch}_test.log
}
echo "hpc_build_clawpack - install clawpack"
hpc_build_clawpack() {
echo "hpc_build_clawpack"
hpc_prep
cd ${simdir}/clawpack
pip install -e . \
2>&1 | tee ${logdir}/clawpack_make_install.log
}
echo "hpc_build_paraview - install paraview"
hpc_build_paraview() {
cd ${srcdir}
}
echo "**************************************************************"
echo -e "---- hpc_build_all - run all install scripts ----"
echo -e " ##### YOU SHOULD RUN hpc_get_sources first! #####"
hpc_build_all() {
hpc_prep
hpc_get_sources
hpc_build_anaconda
hpc_build_mpich
hpc_build_psutil
hpc_build_profiler
hpc_build_cython
hpc_build_mpi4py
hpc_build_petsc
hpc_build_petsc4py
hpc_build_clawpack
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment