Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
Bash script to setup python jupyter notebook with Cloudera Spark2
#!/usr/bin/env bash
set -x -e
test -d ~/venv_notebooks || python -m virtualenv venv_notebooks --python=python$PYTHON_VERSION
. venv_notebooks/bin/activate
python -m pip install --upgrade pip
python -m pip install jupyter ipykernel
echo "Setup jupyter env"
mkdir -p ~/.jupyter
touch ~/.jupyter/
HASHED_PASSWORD=$(python -c "from notebook.auth import passwd; print(passwd('$JUPYTER_PASSWORD'))")
echo "c.NotebookApp.password = u'$HASHED_PASSWORD'" >> ~/.jupyter/
echo "c.NotebookApp.ip = '*'" >> ~/.jupyter/
echo "c.NotebookApp.enable_mathjax = True" >> ~/.jupyter/
echo "c.NotebookApp.open_browser = False" >> ~/.jupyter/
echo "c.IPKernelApp.pylab = 'inline'" >> ~/.jupyter/
echo "Setting ipykernel env..."
python -m ipykernel install --user
mkdir -p ~/.local/share/jupyter/kernels/pyspark2
cat <<EOF > ~/.local/share/jupyter/kernels/pyspark2/kernel.json
{"display_name": "PySpark2", "language": "python","argv": ["$HOME/venv_notebooks/bin/python","-m","ipykernel_launcher","-f","{connection_file}"],"env": {"PYSPARK_PYTHON": "/usr/bin/python","SPARK_HOME": "/opt/cloudera/parcels/SPARK2/lib/spark2","HADOOP_CONF_DIR": "/etc/spark2/conf/yarn-conf","PYTHONPATH": "/opt/cloudera/parcels/SPARK2/lib/spark2/python/lib/","PYTHONSTARTUP": "/opt/cloudera/parcels/SPARK2/lib/spark2/python/pyspark/","PYSPARK_SUBMIT_ARGS": " --master yarn --deploy-mode client pyspark-shell"}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment