Creating Singularity container with Apache Arrow and Spark

Singularity

BootStrap: debootstrap
OSVersion: bionic
MirrorURL: http://us.archive.ubuntu.com/ubuntu/


%runscript
    echo "This is what happens when you run the container..."
    #/home/tahmad/bulk/images/adam.sh

%post
    echo "Hello from inside the container"
    sed -i 's/$/ universe/' /etc/apt/sources.list
    apt-get update
    apt-get -y install vim
    apt-get -y update
    apt-get -y upgrade
    apt-get -y dist-upgrade
    apt-get -y install vim
    apt-get install -y curl
    apt-get install cmake -y
    apt-get update; apt-get install python-pip -y
    pip install psutil
    apt-get install -y software-properties-common
    #apt-get install add-apt-repository
    add-apt-repository ppa:chris-lea/redis-server
    apt-get update
    apt-get install flex bison -y
    apt-get install libboost-all-dev -y 
    apt-get update && apt-get install -y python
    apt-get update && apt install -y python-pip
    apt-get update && pip install numpy
    apt-get update && pip install pyarrow==0.12.0 
    apt-get update && pip install pandas
    apt-get update && apt install -y maven
    apt-get update && apt-get install -y openjdk-8-jre
    apt-get update && apt-get install -y openjdk-8-jdk
    echo 2 | update-alternatives --config java    
    #apt-get update && apt-get install -y default-jre
    #apt-get update && apt-get install -y default-jdk
    apt-get update && apt-get install -y gtk-doc-tools
    apt-get update && apt-get install -y libglib2.0-dev
    apt-get update && apt-get install -y libtool-bin
    apt-get update && apt-get install -y autoconf
    apt-get update && apt-get install -y autoconf-archive
    apt-get update && apt-get install -y libgirepository1.0-dev
    apt-get update && apt install -y git
    apt-get update && apt-get install -y linux-tools-generic
    apt-get install ca-certificates-java 
    apt-get install sed
    apt-get install wget


   
    #curl -O https://archive.apache.org/dist/spark/spark-2.3.4/spark-2.3.4-bin-hadoop2.7.tgz
    #tar xvf spark-2.3.4-bin-hadoop2.7.tgz
    #mv spark-2.3.4-bin-hadoop2.7/ /opt/spark
    #rm  spark-2.3.4-bin-hadoop2.7.tgz
    

   
    #rm -r arrow/
    #git clone https://github.com/apache/arrow.git
    #git clone --single-branch --branch maint-0.12.x https://github.com/tahashmi/arrow.git
    #cd arrow/cpp
    #mkdir release
    #cd release
    #cmake -DCMAKE_BUILD_TYPE=Release -DARROW_PARQUET=ON -DARROW_PLASMA=ON -DARROW_PLASMA_JAVA_CLIENT=ON -DARROW_PYTHON=ON ..
    #make install

    
    
    #cd /arrow/java
    #git submodule update --init --recursive # Needed for flight
    #mvn install -Djavax.net.ssl.trustStorePassword=changeit

    #cd arrow/c_glib
    #./autogen.sh
    #./configure
    #make
    #make install

    apt-get clean

%environment
    export SPARK_HOME=/home/zalars/tahmad/spark 
    #export SPARK_HOME=/home/tahmad/tahmad/spark-2.3.4-bin-hadoop2.7
    chmod -R a+rwx $SPARK_HOME

    #export SPARK_HOME=/opt/spark
    #export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

    #export ADAM_HOME=/opt/adam
    #export PATH=$PATH:$ADAM_HOME/bin:$ADAM_HOME/bin

    export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
    export PATH=$JAVA_HOME/bin:$PATH

    export PATH=$PATH:/usr/local/bin
    #JAVA_HOME=/usr/bin/java
    #export TOOLS_PREFIX='/usr/local'
    export GCC_VERSION='5.5.0'
    export BOOST_VERSION='1_67_0'
    export CPATH='/usr/local/include:/usr/lib'
    export PKG_CONFIG_PATH='/usr/local/lib/pkgconfig'
    export LD_LIBRARY_PATH='/usr/lib:/usr/lib/x86_64-linux-gnu:/usr/local/lib:/usr/lib/x86_64-linux-gnu/openmpi/lib'
    #export PATH=$TOOLS_PREFIX/bin:$PATH
    #echo $LD_LIBRARY_PATH


    #python3.7 -m pip install pandas
    #python3.7 -m pip install numpy
    
    #commands
    #sudo singularity build --sandbox bionic Singularity
    #sudo singularity build --writable bionic.simg bionic

    #sudo singularity build --writable spark.simg Singularity