Bash script to setup python jupyter notebook with Cloudera Spark2
#!/usr/bin/env bash | |
set -x -e | |
JUPYTER_PASSWORD=${1:-myJupyterPassword} | |
PYTHON_VERSION=${2:-3.5} | |
test -d ~/venv_notebooks || python -m virtualenv venv_notebooks --python=python$PYTHON_VERSION | |
. venv_notebooks/bin/activate | |
python -m pip install --upgrade pip | |
python -m pip install jupyter ipykernel | |
echo "Setup jupyter env" | |
mkdir -p ~/.jupyter | |
touch ~/.jupyter/jupyter_notebook_config.py | |
HASHED_PASSWORD=$(python -c "from notebook.auth import passwd; print(passwd('$JUPYTER_PASSWORD'))") | |
echo "c.NotebookApp.password = u'$HASHED_PASSWORD'" >> ~/.jupyter/jupyter_notebook_config.py | |
echo "c.NotebookApp.ip = '*'" >> ~/.jupyter/jupyter_notebook_config.py | |
echo "c.NotebookApp.enable_mathjax = True" >> ~/.jupyter/jupyter_notebook_config.py | |
echo "c.NotebookApp.open_browser = False" >> ~/.jupyter/jupyter_notebook_config.py | |
echo "c.IPKernelApp.pylab = 'inline'" >> ~/.jupyter/jupyter_notebook_config.py | |
echo "Setting ipykernel env..." | |
python -m ipykernel install --user | |
mkdir -p ~/.local/share/jupyter/kernels/pyspark2 | |
cat <<EOF > ~/.local/share/jupyter/kernels/pyspark2/kernel.json | |
{"display_name": "PySpark2", "language": "python","argv": ["$HOME/venv_notebooks/bin/python","-m","ipykernel_launcher","-f","{connection_file}"],"env": {"PYSPARK_PYTHON": "/usr/bin/python","SPARK_HOME": "/opt/cloudera/parcels/SPARK2/lib/spark2","HADOOP_CONF_DIR": "/etc/spark2/conf/yarn-conf","PYTHONPATH": "/opt/cloudera/parcels/SPARK2/lib/spark2/python/lib/py4j-0.10.7-src.zip:/opt/cloudera/parcels/SPARK2/lib/spark2/python/","PYTHONSTARTUP": "/opt/cloudera/parcels/SPARK2/lib/spark2/python/pyspark/shell.py","PYSPARK_SUBMIT_ARGS": " --master yarn --deploy-mode client pyspark-shell"}} | |
EOF |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment