This is version 1.0 of the script to install Pyclaw and dependencies on Shaheen

pyclaw_install_shaheen.sh

# This is version 2.0 of the script to install Pyclaw and dependencies on Shaheen
# Authors: Aron Ahmadia, Damian San Roman
# Collaborators: David Ketcheson, Lisandro Dalcin
# ------------------------------------------------------------------------------ #
# This version uses github and bitbucket sources, alternatively you may select
# non-git sources, supported via wget
# ------------------------------------------------------------------------------ #
# ------------------------------------------------------------------------------ #

echo "\nPrep/Setup"
echo "~~~~~~~~~~"

echo "pyclaw_prep               - set up build environment"

custom_riemann_prep() {
  pyclaw_prep
  customriemanndir=~/devel/sandbox/emclaw/maxwell_vc_3d
  echo "custom riemann = ${customriemanndir}"
}

pyclaw_prep() {
  echo "pyclaw prerp"
  projdir=/home/sanromd/dev
  echo "projdir=${projdir}"
  sandbox=${projdir}/sandbox
  echo "sandbox=${sandbox}"
  builddir=${projdir}/opt/share
  echo "builddir=${builddir}"
  logdir=${builddir}/logs
  echo "logdir=${logdir}"
  srcdir=${builddir}/sources
  echo "srcdir=${srcdir}"

  psutil_ver=2.0.0
  np_ver=1.6.2
  sp_ver=0.13.3
  nose_ver=1.3.0
  py_ver=2.7.2
  zlib_ver=1.2.6
  bzip2_ver=1.0.6
  petsc_ver=3.3-p7
  mprofiler_ver=0.31
  cython_ver=master
  mpi4py_ver=master
  petsc_branch=maint
  petsc_subdir=maint
  petsc4py_branch=maint
  git_source=1
  # echo "python        version   ${py_ver}"
  # echo "numpy         version   ${np_ver}"
  # echo "zlib          version   ${zlib_ver}"
  # echo "bzip2         version   ${bzip2_ver}"
  # echo "scipy         version   ${sp_ver}"
  # echo "psutil        version   ${psutil_ver}"
  # echo "mem profiler  version   ${mprofiler_ver}"
  # echo "nose          version   ${nose_ver}"

  mkdir -p ${sandbox} ${builddir} ${logdir} ${srcdir}

  #disable_threads="yes"

  #if [ disable_threads == "yes" ] ; then
  #threads_flag="--without-threads"
  #  echo "Python and extension modules will be built without thread support"
  #else
  threads_flag="--with-threads"
  #  echo "Python and extension modules will be built with thread support"
  #fi
  module load ibm
  #module load hdf5/1.8.9/ibm
  pic_flag="-fpic"
}

echo "pyclaw_build_ppc64_python             - install python ${py_ver}"
pyclaw_build_ppc64_python() {
  echo "pyclaw_prep"
  pyclaw_prep
  cd ${sandbox}
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/Python-${py_ver}.tgz ]; then 
      wget --no-check-certificate -P ${srcdir} http://www.python.org/ftp/python/${py_ver}/Python-${py_ver}.tgz
  fi
  if [ ! -f $srcdir/Python-${py_ver}_ppc64.patch ]; then 
      wget --no-check-certificate -P ${srcdir} http://dl.dropbox.com/u/65439/Python-${py_ver}_ppc64.patch
  fi

  tar -zxvf ${srcdir}/Python-${py_ver}.tgz
  cd Python-${py_ver}

  echo "source patches"
  patch -p1 < ${srcdir}/Python-${py_ver}_ppc64.patch

  ./configure --prefix=${builddir}/python/${py_ver}/ppc64 --enable-shared \
      --disable-ipv6 --enable-unicode=ucs2 $threads_flag \
      2>&1 | tee ${logdir}/Python-${py_ver}_ppc64_configure.log

  make 2>&1 | tee ${logdir}/Python-${py_ver}_ppc64_make.log  
  make install 2>&1 | tee ${logdir}/Python-${py_ver}_ppc64_make_install.log
}

echo "pyclaw_build_numpy_ppc64              - install numpy ${np_ver}"
pyclaw_build_numpy_ppc64() {
  echo "k01 prep"
  pyclaw_prep
  cd ${sandbox}
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/numpy-${np_ver}.tar.gz ]; then 
      wget --no-check-certificate -P ${srcdir} http://garr.dl.sourceforge.net/project/numpy/NumPy/${np_ver}/numpy-${np_ver}.tar.gz
  fi
  if [ ! -f $srcdir/numpy-${np_ver}_bgp.patch ]; then 
      wget --no-check-certificate -P ${srcdir} http://dl.dropbox.com/u/65439/numpy-1.6.2_bgp.patch
  fi

  tar -zxvf ${srcdir}/numpy-${np_ver}.tar.gz
  cd numpy-${np_ver}
  echo "source patches"
  patch -p1 < ${srcdir}/numpy-${np_ver}_bgp.patch

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v build --compiler=unix --fcompiler=gfortran        \
      2>&1 | tee ${logdir}/numpy-${np_ver}_build.log  

  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v install --home=${builddir}/numpy/${np_ver}/ppc64       \
      2>&1 | tee ${logdir}/numpy-${np_ver}_install.log  
}

echo "\n Target environment (BG/P) scripts"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"

echo "pyclaw_build_zlib                     - install zlib ${zlib_ver}"
pyclaw_build_zlib() {
  echo "pyclaw_prep"
  pyclaw_prep
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/zlib-${zlib_ver}.tgz ]; then 
      wget --no-check-certificate -P $srcdir http://prdownloads.sourceforge.net/libpng/zlib-${zlib_ver}.tar.gz
  fi

  cd $sandbox

  tar -zxvf $srcdir/zlib-${zlib_ver}.tar.gz
  cd zlib-${zlib_ver}
  CC=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-gcc \
      CFLAGS=" $pic_flag " \
      ./configure --prefix=${builddir}/zlib/${zlib_ver}/bgp \
      2>&1 | tee ${logdir}/zlib-${zlib_ver}_bgp_configure.log
  make 2>&1 | tee ${logdir}/zlib-${zlib_ver}_bgp_make.log  
  make install 2>&1 | tee ${logdir}/zlib-${zlib_ver}_bgp_make_install.log
}

echo "pyclaw_build_bzip2                    - install bzip2 ${bzip2_ver}"
pyclaw_build_bzip2() {
  echo "pyclaw_prep"
  pyclaw_prep
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/bzip2-${bzip2_ver}.tgz ]; then 
      wget --no-check-certificate -P $srcdir http://bzip.org/${bzip2_ver}/bzip2-${bzip2_ver}.tar.gz
  fi

  cd $sandbox

  tar -zxvf $srcdir/bzip2-${bzip2_ver}.tar.gz
  cd bzip2-${bzip2_ver}

  make install PREFIX=${builddir}/bzip2/${bzip2_ver}/bgp \
      CC=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-gcc \
      CFLAGS="-Wall -Winline -O2 -g $pic_flag -D_FILE_OFFSET_BITS=64" \
      2>&1 | tee ${logdir}/bzip2-${bzip2_ver}_bgp_make_install.log
}

echo "pyclaw_build_bgp_python               - install python ${py_ver}"
pyclaw_build_bgp_python() {
  echo "pyclaw_prep"
  pyclaw_prep
  cd ${sandbox}
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/Python-${py_ver}.tgz ]; then 
      wget --no-check-certificate -P ${srcdir} http://www.python.org/ftp/python/${py_ver}/Python-${py_ver}.tgz
  fi
  if [ ! -f $srcdir/Python-${py_ver}_bgp.patch ]; then 
      wget --no-check-certificate -P ${srcdir} http://dl.dropbox.com/u/65439/Python-${py_ver}_bgp.patch
  fi

  tar -zxvf ${srcdir}/Python-${py_ver}.tgz
  cd Python-${py_ver}
  echo "build host Python"
  ./configure 2>&1 | tee ${logdir}/Python-${py_ver}_host_configure.log
  make python Parser/pgen 2>&1 | tee ${logdir}/Python-${py_ver}_host_make.log
  mv python hostpython
  mv Parser/pgen Parser/hostpgen
  make distclean

  echo "source patches"
  patch -p1 < ${srcdir}/Python-${py_ver}_bgp.patch

  CFLAGS="-dynamic" \
      CXXFLAGS="-dynamic" \
      CPPFLAGS="-I${builddir}/zlib/${zlib_ver}/bgp/include \
      -I${builddir}/bzip2/${bzip2_ver}/bgp/include" \
      LDFLAGS="-L${builddir}/zlib/${zlib_ver}/bgp/lib -L${builddir}/bzip2/${bzip2_ver}/bgp/lib " \
      CC=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-gcc  \
      CXX=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-g++ \
      AR=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-ar \
      RANLIB=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-ranlib \
      ./configure --host=ppc-linux --build=ppc-linux-gnu --prefix=${builddir}/python/${py_ver}/bgp \
      --enable-shared --disable-ipv6 --enable-unicode=ucs2 $threads_flag \
      2>&1 | tee ${logdir}/Python-${py_ver}_bgp_configure.log

  make HOSTPYTHON=./hostpython HOSTPGEN=./Parser/hostpgen \
      BLDSHARED="/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-gcc -shared" \
      CROSS_COMPILE=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux- \
      CROSS_COMPILE_TARGET=yes HOSTARCH=ppc-linux BUILDARCH=ppc-linux-gnu \
      2>&1 | tee ${logdir}/Python-${py_ver}_bgp_make.log

  make install HOSTPYTHON=./hostpython \
      BLDSHARED="/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux-gcc -shared" \
      CROSS_COMPILE=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/bin/powerpc-bgp-linux- \
      CROSS_COMPILE_TARGET=yes prefix=${builddir}/python/${py_ver}/bgp \
      2>&1 | tee ${logdir}/Python-${py_ver}_bgp_make_install.log
}

echo "pyclaw_build_numpy                    - install numpy ${np_ver}"
pyclaw_build_numpy() {
  echo "pyclaw_build_numpy"
  pyclaw_prep
  cd ${sandbox}

  if [ -d $sandbox/numpy-${np_ver} ]; then
      rm -rf numpy-${np_ver}
  fi

  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/numpy-${np_ver}.tar.gz ]; then 
      wget --no-check-certificate -P ${srcdir} http://garr.dl.sourceforge.net/project/numpy/NumPy/${np_ver}/numpy-${np_ver}.tar.gz
  fi
  if [ ! -f $srcdir/numpy-${np_ver}_bgp.patch ]; then 
      wget --no-check-certificate -P ${srcdir} http://dl.dropbox.com/u/65439/numpy-${np_ver}_bgp.patch
  fi

  tar -zxvf ${srcdir}/numpy-${np_ver}.tar.gz
  cd numpy-${np_ver}
  echo "source patches"
  patch -p1 < ${srcdir}/numpy-${np_ver}_bgp.patch

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v build --compiler=mpixlc --fcompiler=bgp           \
      2>&1 | tee ${logdir}/numpy-${np_ver}_build.log  

  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v install --home=${builddir}/numpy/${np_ver}/bgp        \
      2>&1 | tee ${logdir}/numpy-${np_ver}_install.log  
  
}

echo "pyclaw_build_scipy                    - install scipy ${sp_ver}"
pyclaw_build_scipy() {
  echo "pyclaw_build_scipy"
  pyclaw_prep
  cd ${sandbox}

  if [ -d $sandbox/scipy-${sp_ver} ]; then
      rm -rf scipy-${sp_ver}
  fi

  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/numpy-${np_ver}.tar.gz ]; then
      wget --no-check-certificate -P ${srcdir} http://garr.dl.sourceforge.net/project/scipy/scipy/${sp_ver}/scipy-${sp_ver}.tar.gz
  fi


  tar -zxvf ${srcdir}/scipy-${sp_ver}.tar.gz
  cd scipy-${sp_ver}

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v build --compiler=mpixlc --fcompiler=bgp           \
      2>&1 | tee ${logdir}/scipy-${sp_ver}_build.log

  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v install --home=${builddir}/scipy/${sp_ver}/bgp        \
      2>&1 | tee ${logdir}/scipy-${sp_ver}_install.log

}

echo "pyclaw_build_psutil                   - install psutils ${psutil_ver}"
pyclaw_build_psutil() {
  echo "pyclaw_build_psutil"
  pyclaw_prep
  cd ${sandbox}
  if [ -d $sandbox/psutil-${psutil_ver} ]; then
      rm -rf psutil-${psutil_ver}
  fi

  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/psutil-${psutil_ver} ]; then
    wget --no-check-certificate --no-check-certificate -P ${srcdir} https://pypi.python.org/packages/source/p/psutil/psutil-${psutil_ver}.tar.gz
  fi

  tar -zxvf ${srcdir}/psutil-${psutil_ver}.tar.gz
  cd psutil-${psutil_ver}

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v build --compiler=mpixlc \
      2>&1 | tee ${logdir}/psutil-${psutil_ver}_build.log  

  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v install --home=${builddir}/psutil/${psutil_ver}/bgp        \
      2>&1 | tee ${logdir}/psutil-${psutil_ver}_install.log  

}

echo "pyclaw_build_profiler                 - install memory profiler ${mprofiler_ver}"
pyclaw_build_profiler() {
  echo "pyclaw_build_profiler"
  pyclaw_prep
  cd ${sandbox}

  if [ -d $sandbox/memory_profiler-${mprofiler_ver} ]; then
      rm -rf memory_profiler-${mprofiler_ver}
  fi

  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/memory_profiler-${mprofiler_ver}.tar.gz ]; then
      wget --no-check-certificate --no-check-certificate -P ${srcdir} https://pypi.python.org/packages/source/m/memory_profiler/memory_profiler-${mprofiler_ver}.tar.gz
  fi

  tar -zxvf ${srcdir}/memory_profiler-${mprofiler_ver}.tar.gz
  cd memory_profiler-${mprofiler_ver}

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v build --compiler=mpixlc \
      2>&1 | tee ${logdir}/memory_profiler-${mprofiler_ver}_build.log  

  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v install --home=${builddir}/memory_profiler/${mprofiler_ver}/bgp        \
      2>&1 | tee ${logdir}/memory_profiler-${mprofiler_ver}_install.log  

}

echo "pyclaw_build_nose                     - install nose ${nose_ver}"
pyclaw_build_nose() {
  echo "pyclaw_build_nose"
  pyclaw_prep
  cd ${sandbox}
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/nose-${nose_ver}.tar.gz ]; then 
      wget --no-check-certificate --no-check-certificate -P ${srcdir} http://pypi.python.org/packages/source/n/nose/nose-${nose_ver}.tar.gz
  fi

  tar -zxvf ${srcdir}/nose-${nose_ver}.tar.gz
  cd nose-${nose_ver}

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v build 2>&1 | tee ${logdir}/nose-${nose_ver}_build.log  

  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py \
      -v install --home=${builddir}/nose/${nose_ver}/bgp        \
      2>&1 | tee ${logdir}/nose-${nose_ver}_install.log  
}

echo "pyclaw_test_numpy                     - test numpy build"
pyclaw_test_numpy() {    
  echo "pyclaw_test_numpy"
  pyclaw_prep

  cat <<'EOF' > test_numpy.py
import numpy
numpy.test()
EOF
    
  cat <<'EOF' > test_numpy_submit.ll
#!/usr/bin/env bash
#
# @ job_name            = test_numpy_${np_ver}
# @ job_type            = bluegene
# @ output              = ./$(job_name)_$(jobid).out
# @ error               = ./$(job_name)_$(jobid).err
# @ environment         = COPY_ALL; 
# @ wall_clock_limit    = 0:15:00,0:15:00
# @ notification        = always
# @ bg_size             = 64
# @ account_no          = k193

# @ queue
    
projdir=${projdir}
builddir=${projdir}/opt/share
pythondir=${builddir}/python/${py_ver}/bgp
logdir=${builddir}/logs
sandbox=${projdir}/sandbox
ldpath=${pythondir}/lib
bgp_python_path=${builddir}/numpy/${np_ver}/bgp/lib/python:${builddir}/nose/${nose_ver}/bgp/lib/python

mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
    -mode VN -exp_env HOME -n 1 ${pythondir}/bin/python test_numpy.py | tee ${logdir}/test_numpy.log
EOF
    
  llsubmit test_numpy_submit.ll
}

echo "pyclaw_test_numpy                     - test scipy build"
pyclaw_test_scipy() {    
  echo "k01 prep"
  pyclaw_prep

  cat <<'EOF' > test_scipy.py
import scipy
scipy.test()
EOF
    
  cat <<'EOF' > test_scipy_submit.ll
#!/usr/bin/env bash
#
# @ job_name            = test_scipy_${sp_ver}
# @ job_type            = bluegene
# @ output              = ./$(job_name)_$(jobid).out
# @ error               = ./$(job_name)_$(jobid).err
# @ environment         = COPY_ALL; 
# @ wall_clock_limit    = 0:15:00,0:15:00
# @ notification        = always
# @ bg_size             = 64
# @ account_no          = k193

# @ queue
    
projdir=${projdir}
builddir=${projdir}/opt/share
pythondir=${builddir}/python/${py_ver}/bgp
logdir=${builddir}/logs
sandbox=${projdir}/sandbox
ldpath=${pythondir}/lib
bgp_python_path=${builddir}/numpy/${np_ver}/bgp/lib/python:${builddir}/scipy/${sp_ver}/bgp/lib/python::${builddir}/nose/${nose_ver}/bgp/lib/python

mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
    -mode VN -exp_env HOME -n 1 ${pythondir}/bin/python test_scipy.py | tee ${logdir}/test_scipy.log
EOF
    
  llsubmit test_scipy_submit.ll
}

echo "pyclaw_build_cython                   - install cython"
pyclaw_build_cython() {
  echo "pyclaw_build_cython"
  pyclaw_prep
  cd ${srcdir}
  if [ ! -d $srcdir/cython ]; then 
    git clone git@github.com:cython/cython.git
  fi
  cd cython
  git checkout ${cython_ver}
  git pull
  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH \
  $PYTHON setup.py -v build_ext \
      install --home=${builddir}/cython/bgp \
    2>&1 | tee ${logdir}/cython_install.log 
}

echo "pyclaw_build_mpi4py                   - install mpi4py ${mpi4py_ver}"
pyclaw_build_mpi4py() {
  echo "pyclaw_build_mpi4py"
  pyclaw_prep
  cd ${srcdir}
  if [ ! -d $srcdir/mpi4py ]; then 
    git clone git@bitbucket.org:mpi4py/mpi4py.git
  fi
  cd mpi4py
  git pull
  git checkout ${mpi4py_ver}
  git pull

  CYTHON=${builddir}/cython/bgp/lib/python
  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python 
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib:$CYTHON
  SAVEPATH=PYTHONPATH
  PYTHONPATH=$CYTHON
  export PYTHONPATH=$PYTHONPATH
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py -v build_src \
      build_ext \
      install --home=${builddir}/mpi4py/bgp \
    2>&1 | tee ${logdir}/mpi4py_install.log
}

echo "pyclaw_build_petsc                    - install petsc-${petsc_ver}"
pyclaw_build_petsc() {
  echo "pyclaw_build_petsc"
  pyclaw_prep
  cd ${sandbox}
  if [ ! -f ${srcdir}/petsc-lite-${petsc_ver}.tar.gz ]; then
      wget --no-check-certificate -P ${srcdir} http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-lite-${petsc_ver}.tar.gz
  fi
  
  tar -zxvf ${srcdir}/petsc-lite-${petsc_ver}.tar.gz
  
  cd petsc-${petsc_ver}

  echo "configuration"

  unset PETSC_DIR
  
  ./configure                                                                         \
      --with-mpi-dir=/bgsys/drivers/ppcfloor/comm                                     \
      --known-mpi-shared-libraries=1                                                  \
      --download-fblaslapack=1                                                        \
      --with-x=0                                                                      \
      --with-is-color-value-type=short                                                \
      -CFLAGS=-g -CXXFLAGS=-g                                                         \
      -FFLAGS=-g --with-debugging=1                                                   \
      -COPTFLAGS=-g                                                                   \
      -FOPTFLAGS=-g                                                                   \
      --with-batch=1 --known-memcmp-ok --known-sizeof-char=1                          \
      --with-64-bit-indices=1                                                         \
      --known-sizeof-void-p=4 --known-sizeof-short=2 --known-sizeof-int=4             \
      --known-sizeof-long=4 --known-sizeof-size_t=4 --known-sizeof-long-long=8        \
      --known-sizeof-float=4 --known-sizeof-double=8 --known-bits-per-byte=8          \
      --known-sizeof-MPI_Comm=4 --known-sizeof-MPI_Fint=4 --known-mpi-long-double=1   \
      --known-level1-dcache-assoc=0 --known-level1-dcache-linesize=32                 \
      --known-level1-dcache-size=32768 --known-complex-dot-arg=0                      \
      --download-make                                                                 \
      --download-cmake                                                                \
      --with-shared-libraries=1                                                       \
      --with-batch=1                                                                  \
      LIBS=-dynamic                                                                   \
      PETSC_ARCH=arch-gnu-so                                                          \
      --with-etags=0                                                                  \
      --prefix=${builddir}/petsc/${petsc_ver}/bgp                                  \
      2>&1 | tee ${logdir}/petsc-${petsc_ver}_configure.log

  make PETSC_DIR=${srcdir}/petsc PETSC_ARCH=arch-gnu-so PETSC_ARCH=arch-gnu-so all \
      2>&1 | tee ${logdir}/petsc-${petsc_ver}_make.log

  make PETSC_DIR=/home/sanromd/dev/opt/share/sources/petsc PETSC_ARCH=arch-gnu-so install \
      2>&1 | tee ${logdir}/petsc-${petsc_ver}_make_install.log

  export PETSC_DIR=${builddir}/petsc/${petsc_ver}/bgp
}

echo "pyclaw_build_petsc_git                - install petsc via git"
pyclaw_build_petsc_git() {
  echo "pyclaw_build_petsc_git"
  pyclaw_prep
  cd ${srcdir}
  if [ ! -d $srcdir/petsc ]; then 
    git clone git@bitbucket.org:petsc/petsc.git petsc
  fi
 
  cd petsc
  git checkout ${petsc_branch}
  git pull

  echo "configuration"
  
  unset PETSC_DIR
  
  ./configure                                                                         \
      --with-mpi-dir=/bgsys/drivers/ppcfloor/comm                                     \
      --known-mpi-shared-libraries=1                                                  \
      --download-fblaslapack=1                                                        \
      --with-x=0                                                                      \
      --with-is-color-value-type=short                                                \
      -CFLAGS=-g -CXXFLAGS=-g                                                         \
      -FFLAGS=-g --with-debugging=1                                                   \
      -COPTFLAGS=-g                                                                   \
      -FOPTFLAGS=-g                                                                   \
      --with-batch=1 --known-memcmp-ok --known-sizeof-char=1                          \
      --with-64-bit-indices=1                                                         \
      --known-sizeof-void-p=4 --known-sizeof-short=2 --known-sizeof-int=4             \
      --known-sizeof-long=4 --known-sizeof-size_t=4 --known-sizeof-long-long=8        \
      --known-sizeof-float=4 --known-sizeof-double=8 --known-bits-per-byte=8          \
      --known-sizeof-MPI_Comm=4 --known-sizeof-MPI_Fint=4 --known-mpi-long-double=1   \
      --known-level1-dcache-assoc=0 --known-level1-dcache-linesize=32                 \
      --known-level1-dcache-size=32768 --known-complex-dot-arg=0                      \
      --download-make                                                                 \
      --download-cmake                                                                \
      --with-shared-libraries=1                                                       \
      --with-batch=1                                                                  \
      LIBS=-dynamic                                                                   \
      PETSC_ARCH=arch-gnu-so                                                          \
      --with-etags=0                                                                  \
      --prefix=${builddir}/petsc/${petsc_branch}/bgp                                  \
      2>&1 | tee ${logdir}/petsc-${petsc_branch}_configure.log

  make PETSC_DIR=${srcdir}/petsc PETSC_ARCH=arch-gnu-so PETSC_ARCH=arch-gnu-so all \
      2>&1 | tee ${logdir}/petsc-${petsc_branch}_make.log

  make PETSC_DIR=/home/sanromd/dev/opt/share/sources/petsc PETSC_ARCH=arch-gnu-so install \
      2>&1 | tee ${logdir}/petsc-${petsc_branch}_make_install.log

  export PETSC_DIR=${builddir}/petsc/${petsc_branch}/bgp
}

echo "pyclaw_build_petsc4py_git             - install petsc4py via git"
pyclaw_build_petsc4py_git() {
  echo "pyclaw_build_petsc4py_git"
  pyclaw_prep
  cd ${srcdir}
  if [ ! -d $srcdir/petsc4py ]; then 
    git clone -b ${petsc4py_branch} git@bitbucket.org:petsc/petsc4py.git petsc4py
  fi
  cd petsc4py
  git pull
 
  export PETSC_DIR=${builddir}/petsc/${petsc_branch}/bgp

  CYTHON=${builddir}/cython/bgp/lib/python
  SAVEPATH=PYTHONPATH
  PYTHONPATH=$CYTHON
  export PYTHONPATH=$PYTHONPATH

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib:$CYTHON
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py -v build_src build_ext \
      --include-dirs=${builddir}/numpy/${np_ver}/bgp/lib/python/numpy/core/include \
      install --home=${builddir}/petsc4py/${petsc4py_branch}/bgp
}


echo "pyclaw_build_petsc4py                 - install petsc4py ${petsc4py_ver}"
pyclaw_build_petsc4py() {
  echo "pyclaw_build_petsc4py"
  pyclaw_prep
  cd ${sandbox}
  echo "downloading and unpacking sources"
  if [ ! -f $srcdir/petsc4py-${petsc4py_ver}.tar.gz ]; then
      wget --no-check-certificate --no-check-certificate -P ${srcdir} https://petsc4py.googlecode.com/files/petsc4py-${petsc4py_ver}.tar.gz
  fi

  tar -zxvf ${srcdir}/petsc4py-${petsc4py_ver}.tar.gz
  cd petsc4py-${petsc4py_ver}
  
  export PETSC_DIR=${builddir}/petsc/${petsc_ver}/bgp

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python
  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH $PYTHON setup.py -v build_ext \
      --compiler=mpixlc \
      --include-dirs=${builddir}/numpy/${np_ver}/bgp/lib/python/numpy/core/include \
      install --home=${builddir}/petsc4py/${petsc4py_ver}/bgp
}

echo "pyclaw_build_clawpack                 - install clawpack"
pyclaw_build_clawpack() {
  echo "pyclaw_build_clawpack"
  pyclaw_prep

  cd ${sandbox}
  git clone git://github.com/clawpack/clawpack.git
  cd clawpack/
  git submodule init
  git submodule update
  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python

  SAVEPATH=$PYTHONPATH
  export PYTHONPATH=${builddir}/numpy/${np_ver}/ppc64/lib/python

  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH \
    $PYTHON \
    setup.py -v build_ext \
    --fcompiler=bgp \
    install --home=${builddir}/clawpack/dev/bgp
  export PYTHONPATH=$SAVEPATH
}

echo "pyclaw_rebuild_clawpack_with_current_branches"
pyclaw_rebuild_clawpack_with_current_branches() {
  echo "pyclaw_rebuild_clawpack_with_current_branches"
  pyclaw_prep

  cd ${sandbox}
  cd clawpack/
  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python

  SAVEPATH=$PYTHONPATH
  export PYTHONPATH=${builddir}/numpy/${np_ver}/ppc64/lib/python

  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH \
    $PYTHON \
    setup.py -v build_ext \
    --compiler=mpixlc --fcompiler=bgp \
    install --home=${builddir}/clawpack/dev/bgp
  export PYTHONPATH=$SAVEPATH
}

echo "pyclaw_rebuild_clawpack"
pyclaw_rebuild_clawpack() {
  echo "pyclaw_rebuild_clawpack"
  pyclaw_prep

  cd ${sandbox}
  cd clawpack/
  git checkout ${branch_name}
  git submodule init
  git submodule update
  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python

  SAVEPATH=$PYTHONPATH
  export PYTHONPATH=${builddir}/numpy/${np_ver}/ppc64/lib/python

  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH \
    $PYTHON \
    setup.py -v build_ext \
    --compiler=mpixlc --fcompiler=bgp \
    install --home=${builddir}/clawpack/dev/bgp
  export PYTHONPATH=$SAVEPATH
}

echo "pyclaw_test_build                     - test pyclaw build"
pyclaw_test_build() {    
  echo "pyclaw_test_build"
  pyclaw_prep


  cat <<'EOF' > test_pyclaw_submit.ll
#!/usr/bin/env bash
#
# @ job_name            = test_pyclaw
# @ job_type            = bluegene
# @ output              = ./$(job_name)_$(jobid).out
# @ error               = ./$(job_name)_$(jobid).err
# @ environment         = COPY_ALL; 
# @ wall_clock_limit    = 0:15:00,0:15:00
# @ notification        = always
# @ bg_size             = 64
# @ account_no          = k47

# @ queue


projdir=~/dev
builddir=${projdir}/opt/share
pythondir=${builddir}/python/${py_ver}/bgp
ldpath=${pythondir}/lib
numpy_path=${builddir}/numpy/${np_ver}/bgp/lib/python
nose_path=${builddir}/nose/1.1.2/bgp/lib/python
clawpack_path=${builddir}/clawpack/dev/bgp/lib/python
petsc4py_path=${builddir}/petsc4py/${petsc4py_ver}/bgp/lib/python
mpi4py_path=${builddir}/mpi4py/bgp/lib/python
bgp_python_path=${numpy_path}:${nose_path}:${clawpack_path}:${petsc4py_path}:${mpi4py_path}
sandbox=${projdir}/sandbox
ldpath=${pythondir}/lib
logdir=${builddir}/logs

mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
    -mode VN -exp_env HOME -n 4 ${pythondir}/bin/python \
    ${sandbox}/clawpack/pyclaw/apps/advection_1d/advection.py \
    kernel_language=Fortran use_petsc=True \
    | tee ${logdir}/test_clawpack_advection.log
EOF

llsubmit test_pyclaw_submit.ll
}

echo "pyclaw_build_custom_riemann"
pyclaw_build_custom_riemann() {
  echo "pyclaw_build_custom_riemann"
  pyclaw_prep
  custom_riemann_prep

  cd ${customriemanndir}

  PYTHON=${builddir}/python/${py_ver}/ppc64/bin/python

  SAVEPATH=$PYTHONPATH
  export PYTHONPATH=${builddir}/numpy/${np_ver}/ppc64/lib/python

  PYTHON_LD_LIBRARY_PATH=${builddir}/python/${py_ver}/ppc64/lib
  LD_LIBRARY_PATH=$PYTHON_LD_LIBRARY_PATH \
    $PYTHON \
    setup.py -v build_ext -i
  export PYTHONPATH=$SAVEPATH
}

echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
echo "pyclaw_build_all - run all install scripts"
pyclaw_build_all() {
  pyclaw_build_ppc64_python
  pyclaw_build_numpy_ppc64
  pyclaw_build_zlib
  pyclaw_build_bzip2
  pyclaw_build_bgp_python
  pyclaw_build_numpy
  pyclaw_build_nose
  pyclaw_build_psutil
  pyclaw_build_profiler
  pyclaw_build_cython
  pyclaw_build_mpi4py
  pyclaw_build_petsc_git
  pyclaw_build_petsc4py_git
  pyclaw_build_clawpack
}

pyclaw_run_tests() {
  pyclaw_test_numpy
  pyclaw_test_build
}