Bash script to setup python jupyter notebook with Cloudera Spark2

jupyter_notebook_cloudera_setup.sh

#!/usr/bin/env bash

set -x -e

JUPYTER_PASSWORD=${1:-myJupyterPassword}
PYTHON_VERSION=${2:-3.5}

test -d ~/venv_notebooks || python -m virtualenv venv_notebooks --python=python$PYTHON_VERSION
. venv_notebooks/bin/activate


python -m pip install --upgrade pip
python -m  pip install jupyter ipykernel

echo "Setup jupyter env"

mkdir -p ~/.jupyter
touch ~/.jupyter/jupyter_notebook_config.py
HASHED_PASSWORD=$(python -c "from notebook.auth import passwd; print(passwd('$JUPYTER_PASSWORD'))")

echo "c.NotebookApp.password = u'$HASHED_PASSWORD'" >> ~/.jupyter/jupyter_notebook_config.py
echo "c.NotebookApp.ip = '*'" >> ~/.jupyter/jupyter_notebook_config.py
echo "c.NotebookApp.enable_mathjax = True" >> ~/.jupyter/jupyter_notebook_config.py
echo "c.NotebookApp.open_browser = False" >> ~/.jupyter/jupyter_notebook_config.py
echo "c.IPKernelApp.pylab = 'inline'" >> ~/.jupyter/jupyter_notebook_config.py


echo "Setting ipykernel env..."
python -m ipykernel install --user

mkdir -p ~/.local/share/jupyter/kernels/pyspark2

cat <<EOF > ~/.local/share/jupyter/kernels/pyspark2/kernel.json
{"display_name": "PySpark2", "language": "python","argv": ["$HOME/venv_notebooks/bin/python","-m","ipykernel_launcher","-f","{connection_file}"],"env": {"PYSPARK_PYTHON": "/usr/bin/python","SPARK_HOME": "/opt/cloudera/parcels/SPARK2/lib/spark2","HADOOP_CONF_DIR": "/etc/spark2/conf/yarn-conf","PYTHONPATH": "/opt/cloudera/parcels/SPARK2/lib/spark2/python/lib/py4j-0.10.7-src.zip:/opt/cloudera/parcels/SPARK2/lib/spark2/python/","PYTHONSTARTUP": "/opt/cloudera/parcels/SPARK2/lib/spark2/python/pyspark/shell.py","PYSPARK_SUBMIT_ARGS": " --master yarn --deploy-mode client pyspark-shell"}}
EOF