This is a script for machine learning tool and install jupyter notebook and lab for easy machine learning algorithms developments

install_jupyter.sh

#!/bin/bash
cd /data/inst/anaconda/bin
source activate

# conda update -y -n base -c defaults conda

conda create -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults python=3.6 


conda install -y  -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults \
            "ipython=7.6.1" \         
            "numpy=1.16.4" \
            "scikit-learn=0.20.2" \
            "nbformat=4.*.*" \
            jupyterlab \
            psutil \
            nodejs \
            ipyparallel \
            matplotlib \
            "tqdm=4.35.0" \
            pyzmq \
            cython \
            graphviz \
            numpy \
            matplotlib \
            scipy \
            requests \
            urllib3 \
            pandas \
            six \
            future \
            protobuf \
            zlib \
            boost \
            psutil \
            PyYAML \
            python-crontab \
            ipykernel \
            jupyter_contrib_nbextensions \
            cmake \
            Pillow \
            tensorflow-base \
            openmpi \
            unixodbc 
conda activate jupyter

jupyter notebook --generate-config
# Download Jupter config from gist
wget -O ${HOME}/.jupyter/jupyter_notebook_config.py https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/jupyter_notebook_config.py


# Pip install section
mkdir -p ${HOME}/DLR_Work/libraries/
wget -O ${HOME}/DLR_Work/libraries/requirements-full.txt https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/requirements-full.txt




$HOME/.conda/envs/jupyter/bin/pip install --no-cache-dir --upgrade -r ${HOME}/DLR_Work/libraries/requirements-full.txt
 
 
 # Activate and configure extensions
jupyter contrib nbextension install --user
# nbextensions configurator
jupyter nbextensions_configurator enable --user
# Active nbresuse
jupyter serverextension enable --py nbresuse
# Activate Jupytext
jupyter nbextension enable --py jupytext
# Disable Jupyter Server Proxy
jupyter nbextension disable jupyter_server_proxy/tree
# Configure nbdime
nbdime config-git --enable --global
# Enable useful extensions
jupyter nbextension enable skip-traceback/main
# jupyter nbextension enable comment-uncomment/main
# Do not enable variable inspector: causes trouble: https://github.com/ml-tooling/ml-workspace/issues/10
# jupyter nbextension enable varInspector/main
#jupyter nbextension enable spellchecker/main
jupyter nbextension enable toc2/main
jupyter nbextension enable execute_time/ExecuteTime
jupyter nbextension enable collapsible_headings/main
jupyter nbextension enable codefolding/main
# Activate Jupyter Tensorboard
jupyter tensorboard enable
# Edit notebook config
# comment for now ### cat $HOME/.jupyter/nbconfig/notebook.json | jq '.toc2={"moveMenuLeft": false}' > tmp.$$.json && mv tmp.$$.json $HOME/.jupyter/nbconfig/notebook.json
# Activate qgrid
jupyter nbextension enable --py --sys-prefix qgrid
# Activate Colab support
jupyter serverextension enable --py jupyter_http_over_ws
# Activate Voila Rendering 
# currently not working jupyter serverextension enable voila --sys-prefix
# Enable ipclusters
# comment for now ### ipcluster nbextension enable



# install jupyterlab

# Required for jupytext and matplotlib plugins
jupyter lab build
# jupyterlab installed in requirements section
jupyter labextension install @jupyter-widgets/jupyterlab-manager

jupyter labextension install @jupyterlab/toc
jupyter labextension install jupyterlab_tensorboard
# install jupyterlab git
jupyter labextension install @jupyterlab/git
pip install jupyterlab-git 
jupyter serverextension enable --py jupyterlab_git
# For Matplotlib: https://github.com/matplotlib/jupyter-matplotlib
jupyter labextension install jupyter-matplotlib
# Install jupyterlab language server support
pip install --pre jupyter-lsp
jupyter labextension install @krassowski/jupyterlab-lsp
# For Bokeh
jupyter labextension install jupyterlab_bokeh
# For Plotly
jupyter labextension install @jupyterlab/plotly-extension
jupyter labextension install jupyterlab-chart-editor
# For holoview
jupyter labextension install @pyviz/jupyterlab_pyviz
# Install jupyterlab_iframe - https://github.com/timkpaine/jupyterlab_iframe
pip install jupyterlab_iframe
jupyter labextension install jupyterlab_iframe
jupyter serverextension enable --py jupyterlab_iframe
# Install jupyterlab variable inspector - https://github.com/lckr/jupyterlab-variableInspector
jupyter labextension install @lckr/jupyterlab_variableinspector
# Install jupyterlab code formattor - https://github.com/ryantam626/jupyterlab_code_formatter
jupyter labextension install @ryantam626/jupyterlab_code_formatter
pip install jupyterlab_code_formatter
jupyter serverextension enable --py jupyterlab_code_formatter
# Install go-to-definition extension 
jupyter labextension install @krassowski/jupyterlab_go_to_definition
# Install ipysheet jupyterlab extension
jupyter labextension install ipysheet
# Deprecation and validations:
# Install jupyterlab_templates - https://github.com/timkpaine/jupyterlab_templates
# Does not work currently
# pip install jupyterlab_templates
# jupyter labextension install jupyterlab_templates
# jupyter serverextension enable --py jupyterlab_templates
# Install jupyterlab-data-explorer: https://github.com/jupyterlab/jupyterlab-data-explorer
# alpha version jupyter labextension install @jupyterlab/dataregistry-extension
# Install jupyterlab system monitor: https://github.com/jtpio/jupyterlab-system-monitor
# Activate ipygrid in jupterlab
# Problems with terminal: jupyter labextension install ipyaggrid
# Install qgrid
# Not compatible to jupyterlab 1.x: https://github.com/quantopian/qgrid/issues/261
# DO not install for now jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
    # Install voyagar data grid
# Does not work with 1.1.1: 
jupyter labextension install jupyterlab_voyager
# Too big dependency: https://github.com/InsightSoftwareConsortium/itkwidgets
# Too Big: Install ipyleaflet
pip install --no-cache-dir ipyleaflet
jupyter labextension install jupyter-leaflet
jupyter lab build
# Cleanup
# Clean jupyter lab cache: https://github.com/jupyterlab/jupyterlab/issues/4930
jupyter lab clean
jlpm cache clean


# Create env for tf 1
conda create --name tf_1  -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults --yes --quiet \
    "matplotlib>=3.0.3" \
    "numpy>1.16.7" \
    "scipy>=1.2.1" \
    "pillow>=4.3.0" \
    "scikit-image" \
    python=3.6\
    ipykernel \
    scikit-learn \
    ;\
	source activate tf_1;\
    conda install --yes --quiet -c anaconda -c conda-forge keras "tensorflow-base==1.*.*";\
	python -m ipykernel install --user --name "tf_1" --display-name "Tensorflow 1 latest";\
	source deactivate;

jupyter_notebook_config.py

# Configuration file for jupyter-notebook.
# jupyter notebook --ip=0.0.0.0 --port=8080
#------------------------------------------------------------------------------
# Application(SingletonConfigurable) configuration
#------------------------------------------------------------------------------

from jupyter_core.paths import jupyter_data_dir
import subprocess
import os
import psutil
import errno
import stat

c = get_config()

os.environ["AUTHENTICATE_VIA_JUPYTER"] = "true"
os.environ["WORKSPACE_BASE_URL"] = os.environ["HOME"]  # "HOME"
print(os.getenv("WORKSPACE_BASE_URL"))

# https://jupyter-notebook.readthedocs.io/en/stable/config.html
# c.NotebookApp.ip = '*'
c.NotebookApp.ip = '127.0.0.1'
c.NotebookApp.port = 8090
c.NotebookApp.notebook_dir="./"
c.NotebookApp.notebook_dir= os.environ["HOME"] + '/DLR_Work'
c.NotebookApp.open_browser = True
c.NotebookApp.allow_root=True



c.JupyterApp.answer_yes = True

# set base url if available
base_url = os.getenv("WORKSPACE_BASE_URL", "/")
if base_url is not None and base_url is not "/":
    c.NotebookApp.base_url=base_url

# Do not delete files to trash: https://github.com/jupyter/notebook/issues/3130
c.FileContentsManager.delete_to_trash=False

# Always use inline for matplotlib
c.IPKernelApp.matplotlib = 'inline'


shutdown_inactive_kernels = os.getenv("SHUTDOWN_INACTIVE_KERNELS", "false")
if shutdown_inactive_kernels and shutdown_inactive_kernels.lower().strip() != "false":
    cull_timeout = 172800 # default is 48 hours
    try: 
        # see if env variable is set as timout integer
        cull_timeout = int(shutdown_inactive_kernels)
    except ValueError:
        pass
    
    if cull_timeout > 0:
        print("Activating automatic kernel shutdown after " + str(cull_timeout) + "s of inactivity.")
        # Timeout (in seconds) after which a kernel is considered idle and ready to be shutdown.
        c.MappingKernelManager.cull_idle_timeout = cull_timeout
        # Do not shutdown if kernel is busy (e.g on long-running kernel cells)
        c.MappingKernelManager.cull_busy = False
        # Do not shutdown kernels that are connected via browser, activate?
        c.MappingKernelManager.cull_connected = False

authenticate_via_jupyter = os.getenv("AUTHENTICATE_VIA_JUPYTER", "false")
if authenticate_via_jupyter and authenticate_via_jupyter.lower().strip() != "false":
    # authentication via jupyter is activated

    # Do not allow password change since it currently needs a server restart to accept the new password
    c.NotebookApp.allow_password_change = False

    if authenticate_via_jupyter.lower().strip() == "<generated>":
        # dont do anything to let jupyter generate a token in print out on console
        pass
    # if true, do not set any token, authentication will be activate on another way (e.g. via JupyterHub)
    elif authenticate_via_jupyter.lower().strip() != "true":
        # if not true or false, set value as token
        c.NotebookApp.token = authenticate_via_jupyter
else:
    # Deactivate token -> no authentication
    c.NotebookApp.token=""





base_url = os.getenv("WORKSPACE_BASE_URL", "/")
if base_url is not None and base_url is not "/":
    c.NotebookApp.base_url=base_url
    
    
#c.NotebookApp.ip = '0.0.0.0'

# https://github.com/timkpaine/jupyterlab_iframe
try:
    if not base_url.startswith("/"):
        base_url = "/" + base_url
    # iframe plugin currently needs absolut URLS
    c.JupyterLabIFrame.iframes = [base_url + 'tools/ungit', base_url + 'tools/netdata', base_url + 'tools/vnc', base_url + 'tools/glances', base_url + 'tools/vscode']
except:
    pass

# https://github.com/timkpaine/jupyterlab_templates
WORKSPACE_HOME = os.getenv("WORKSPACE_HOME", "/workspace")
try:
    if os.path.exists(WORKSPACE_HOME + '/templates'):
        c.JupyterLabTemplates.template_dirs = [WORKSPACE_HOME + '/templates']
    c.JupyterLabTemplates.include_default = False
except:
    pass

requirements-full.txt

# Minimum Requirements

## Utilities
httpie==1.0.3 # HTTPie - a CLI, cURL-like tool for humans.
cloudpickle==1.2.2 # Extended pickling support for Python objects
msgpack==0.6.2 # MessagePack (de)serializer.
msgpack-numpy==0.4.4.3 # Numpy data serialization using msgpack
cysignals==1.10.2 # Interrupt and signal handling for Cython
h5py==2.10.0 # Read and write HDF5 files from Python
seaborn==0.9.0 # Python visualization library based on matplotlib  - Basic
SQLAlchemy==1.3.11 # Database Abstraction Library
virtualenv==16.7.8 # Virtual Python Environment builder
pytest==5.3.2 # pytest: simple powerful testing with Python
autopep8==1.4.4 # A tool that automatically formats Python code to conform to the PEP 8 style guide
pylint==2.4.4 # python code static checker
pycodestyle==2.5.0 # Python style guide checker
pydocstyle==5.0.1 # Python docstring style checker
tensorboard==2.0.* # TensorBoard lets you watch Tensors Flow
wandb==0.8.29

## Glances
psutil==5.6.7 # Cross-platform lib for process and system monitoring in Python.
bottle==0.12.18 # Fast and simple WSGI-framework for small web-applications.
netifaces==0.10.9 # Portable network interface information.
py-cpuinfo==5.0.0 # Get CPU info with pure Python 2 & 3
glances==3.1.3 # A cross-platform curses-based monitoring tool
pymdstat==0.4.2 # Python library to parse Linux /proc/mdstat

## Jupyter
jupytext==1.3.0 # Jupyter notebooks as Markdown documents, Julia, Python or R scripts
nbresuse==0.3.3 # Simple Jupyter extension to show how much resources (RAM) your notebook is using
ipympl==0.3.3 # Matplotlib Jupyter Extension
jupyterhub==1.0.0 # JupyterHub: A multi-user server for Jupyter notebooks
remote_ikernel==0.4.6 # Running IPython kernels through batch queues
jupyter_contrib_nbextensions==0.5.1 # A collection of Jupyter nbextensions.
jupyter_nbextensions_configurator==0.4.1 # jupyter serverextension providing configuration interfaces for nbextensions.
# jupyter-tensorboard==0.1.10 # Jupyter notebook integration for tensorboard.
git+https://github.com/cloudrainstar/jupyter_tensorboard.git # Use other version with support for tensorflow 2.X
nbdime==1.1.0 # Diff and merge of Jupyter Notebooks


# requirements-full.txt
#
####### Basics #######
#
## Compatibility

## Object Serialization
joblib==0.14.1 # Lightweight pipelining: using Python functions as pipeline jobs.
dill==0.3.1.1 # dill can be used to store python objects to a file
marshmallow==3.3.0 # A lightweight library for converting complex datatypes to and from native Python datatype
jsonpickle==1.2 # Python library for serializing any arbitrary object graph into JSON

## Utilities
boltons==19.3.0 # When they're not builtins, they're boltons.
toolz==0.10.0 # A functional standard library for Python.
memory_profiler==0.55.0 # A module for monitoring memory usage of a python program
cytoolz==0.10.1 # Cython implementation of Toolz: High performance functional utilities
arrow==0.15.4 # Better dates & times for Python 
python-dateutil==2.8.0 # 2.8.0 required by botocore: Extensions to the standard Python datetime module
click==7.0.0 # Composable command line interface toolkit
thefuck==3.29 # Magnificent app which corrects your previous console command
fire==0.2.1 # A library for automatically generating command line interfaces.
jmespath==0.9.4 # JSON Matching Expressions
marisa-trie==0.7.5 # Static memory-efficient and fast Trie-like structures for Python.
tabulate==0.8.6 # Pretty-print tabular data in Python
patool==1.12 # portable archive file manager - supports various compression formats
smart-open==1.9.0 # Utils for streaming large files (S3, HDFS, gzip, bz2...)
Delorean==1.0.0 # library for manipulating datetimes with ease and clarity
pipenv==2018.11.26 # Python Development Workflow for Humans.
addict==2.2.1 # Addict is a dictionary whose items can be set using both attribute and item syntax.
attrs==19.3.0 # Classes Without Boilerplate
path.py==12.4.0 # A module wrapper for os.path
fs==2.4.11 # Python's filesystem abstraction layer

# Python Development Helpers
python-language-server==0.31.2 # Python Language Server for the Language Server Protocol
yapf==0.29.0 # formatter for Python files
black==19.10b0 # The uncompromising code formatter.
sphinx==2.3.0 # Python documentation generator
nose2==0.9.1  # The successor to nose, based on unittest2 
flake8==3.7.9 # The modular source code checker: pep8, pyflakes and co
pyflakes==2.1.1 # Passive checker of Python programs
mccabe==0.6.1 # McCabe checker, plugin for flake8
bandit==1.6.2 # Bandit is a tool designed to find common security issues in Python code.
pylama==7.7.1 # Code audit tool for python
rope==0.14.0 # A python refactoring library
# Evaluate: prospector (not compatible to flake8)

####### Data Science Libs #######
#

## ML Frameworks
# tensorflow -> installed via conda for mkl support
# keras -> light dependency
# xgboost -> light dependency
# scikit-learn -> light dependency
# cntk -> 200MB -> installer in tools
# mxnet -> 200MB -> installer in tools
# catboost -> 150MB # State-of-the-art open-source gradient boosting on decision trees library.
fastai==1.0.59 # The fastai library simplifies training fast and accurate neural nets using modern best practices.
lightgbm==2.3.1 # A fast, distributed, high performance gradient boosting framework
theano==1.0.4 # Optimizing compiler for evaluating mathematical expressions on CPUs and GPUs.
chainer==7.0.0 # A flexible framework of neural networks
tflearn==0.3.2 # Deep Learning Library featuring a higher-level API for TensorFlow
dm-sonnet==1.35 # Sonnet is a library for building neural networks in TensorFlow.
statsmodels==0.10.2 # Statistical computations and models for Python
vowpalwabbit==8.8.0 # Fast machine learning library for online learning
catalyst==19.12.1 # High-level framework for PyTorch DL & RL research

## Data Handling and Persistence
xarray==0.14.1 # N-D labeled arrays and datasets in Python
csvkit==1.0.4 # A suite of command-line tools for working with CSV, the king of tabular file formats.
sklearn-pandas==1.8.0 # Pandas integration with sklearn
pandas-datareader==0.8.1 # Data readers extracted from the pandas codebase,should be compatible with recent pandas versions
pandasql==0.7.3 # Allows you to query pandas DataFrames using SQL syntax
category_encoders==2.1.0 # A collection sklearn transformers to encode categorical variables as numeric
blaze==0.10.1 # Blaze translates a subset of modified NumPy and Pandas-like syntax to databases and other computing systems. 
geojson==2.5.0 # Python bindings and utilities for GeoJSON
annoy==1.16.2 # Approximate Nearest Neighbors in C++/Python optimized for memory usage and loading/saving to disk.
easydict==1.9 # Access dict values as attributes (works recursively).
swifter==0.297 # A package which efficiently applies any function to a pandas dataframe or series in the fastest available manner
blosc==1.8.3 # Blosc data compressor
xlrd==1.2.0 # Library for developers to extract data from Microsoft Excel (tm) spreadsheet files

## Text Processing / NLP
spacy==2.2.3 # Industrial-strength Natural Language Processing (NLP) with Python and Cytho
gensim==3.8.1 # Python framework for fast Vector Space Modelling
pyfasttext==0.4.6 # cysignals has to be install manually before pyfasttext to avoid compile errors; installing cysignals in this requirements.txt still causes compile error
fuzzywuzzy==0.17.0 #Fuzzy string matching like a boss. It uses Levenshtein Distance to calculate the differences between sequences in a simple-to-use package.
nltk==3.4.5 # Natural Language Toolkit
torchtext==0.4.0 # Data loaders and abstractions for text and NLP
textacy==0.9.1 # Higher-level text processing, built on spaCy
textblob==0.15.3 # Simple, Pythonic text processing. Sentiment analysis, part-of-speech tagging, noun phrase parsing, and more.
flashtext==2.7 # Fast Extract/Replaces keywords in sentences.
scattertext==0.0.2.56 # DO NOT UPDATE v53 not found - Beautiful visualizations of how language differs among document types. 
sentencepiece==0.1.83  # Unsupervised text tokenizer for Neural Network-based text generation.
langid==1.1.6 # Stand-alone language identification system
scrapy==1.8.0 #A high-level Web Crawling and Web Scraping framework
tiny-tokenizer==3.1.0 # Related to flair install issue https://github.com/flairNLP/flair/issues/1327
flair==0.4.4 # A very simple framework for state-of-the-art NLP - 0.4.3 has too specific requirements
pyahocorasick==1.4.0 # a fast and memory efficient library for exact or approximate multi-pattern string search.
ftfy==5.6 # Fixes some problems with Unicode text after the fact
stop-words==2018.7.23 # Get list of common stop words in various languages in Python
stanfordnlp==0.2.0 # Official Stanford NLP Python Library
newspaper3k==0.2.8 # Simplified python article discovery & extraction.
inflect==3.0.2 # Correctly generate plurals, singular nouns, ordinals, indefinite articles; convert numbers to words
transformers==2.2.2 # Repository of pre-trained NLP Transformer models: BERT & RoBERTa, GPT & GPT-2, Transformer-XL, XLNet and XLM
spacy-transformers==0.5.1 # spaCy pipelines for pre-trained BERT and other transformers
finetune==0.8.5 # Scikit-learn style model finetuning for NLP
fast-bert==1.5.2 # Super easy library for BERT based NLP models 

## Image Processing
ImageHash==4.0 # Image Hashing library
imgaug==0.3.0 # Image augmentation library for machine learning
scikit-image==0.16.2 # Image processing routines for SciPy - version 0.14.1 is not compatible with numpy 16
imageio==2.6.1 # Library for reading and writing a wide range of image, video, scientific, and volumetric data formats.
Augmentor==0.2.7 # Image augmentation library for Machine Learning
opencv-python==4.1.2.30 # Wrapper package for OpenCV python bindings.
imageai==2.1.5 # A python library built to empower developers to build applications and systems with self-contained Computer Vision capabilities
# Pillow -> installed via conda

## Audio/Speech Processing
librosa==0.7.1 # Python module for audio and music processing

## Graph
networkx==2.2 # 2.2 version required by hyperopt: Data structures for graphs, digraphs, and multigraphs
dgl==0.4.1 # Deep Graph Library
graph-nets==1.0.5 # Library for building graph networks in Tensorflow and Sonnet.
torch-cluster==1.4.5 # PyTorch Extension Library of Optimized Graph Cluster Algorithms
torch-geometric==1.3.2 # Geometric Deep Learning Extension Library for PyTorch
torchbiggraph==1.0.0 # A distributed system to learn embeddings of large graphs

# Reinforcement learning
tensorforce==0.5.2 # Reinforcement learning for TensorFlow
gym==0.15.4 # The OpenAI Gym: A toolkit for developing and comparing your reinforcement learning agents.
dopamine-rl==2.0.5 # Dopamine: A framework for flexible Reinforcement Learning research

## Time Series
tsfresh==0.13.0 # tsfresh extracts relevant characteristics from time series

## Recommender Systems
scikit-surprise==1.1.0 # An easy-tfo-use library for recommender systems.
lightfm==1.15 # A Python implementation of LightFM, a hybrid recommendation algorithm.
implicit==0.4.2 # Collaborative Filtering for Implicit Datasets
tensorrec==0.26.2 # A TensorFlow recommendation algorithm and framework in Python.

## Hyperparameter Optimization
hyperopt==0.2.2 # Distributed Asynchronous Hyperparameter Optimization - 0.1.2 has a problem with tqdm write
talos==0.6.3  # Talos Hyperparameter Scanner for Keras 
scikit-optimize==0.5.2 # Sequential model-based optimization toolbox.
optuna==0.19.0 # A hyperparameter optimization framework
hpbandster==0.7.4 # HyPerBAND on STERoids, a distributed Hyperband implementation with lots of room for improvement
dragonfly-opt==0.1.4 # Scalable Bayesian optimisation.

## Parallization and Distribution
dask==2.9.0 # Parallel PyData with Task Scheduling
dask-ml==1.1.1 # A library for distributed and parallel machine learning
deap==1.3.0 # Distributed Evolutionary Algorithms in Python
mpi4py==3.0.3 # Python bindings for MPI
horovod==0.18.2 # Distributed training framework for TensorFlow, Keras, PyTorch, and MXNet. - 30B

## ML Workflow 
sacred==0.8.1 # tool to help you configure, organize, log and reproduce experiments 
mlflow==1.4.0 # MLflow: An ML Workflow Tool
knockknock==0.1.6 # Be notified when your training is complete with only two additional lines of code
kaggle==1.5.6 # Official API for Kaggle accessible using a command line tool implemented in Python. 
steppy==0.1.16 # A lightweight, open-source, Python library for fast and reproducible experimentation

## AutoML
adanet==0.8.0 # adanet is a lightweight and scalable TensorFlow AutoML framework for training and deploying adaptive neural networks using the AdaNet algorithm

## Tensorflow Ecosystem
tensorflow-probability==0.9.0 # Probabilistic modeling and statistical inference in TensorFlow
tensorflow-datasets==1.3.2 # tensorflow/datasets is a library of datasets ready to use with TensorFlow.
tensorpack==0.9.8 # Neural Network Toolbox on TensorFlow
tensor2tensor==1.15.4 # Library of deep learning models and datasets designed to make deep learning more accessible and accelerate ML research.
tensorflow-hub==0.7.0 # TensorFlow Hub is a library to foster the publication, discovery, and consumption of reusable parts of machine learning models.
tensorflow-graphics==1.0.0 # A library that contains well defined, reusable and cleanly written graphics related ops and utility functions for TensorFlow.

## Pytorch Ecosystem: https://pytorch.org/ecosystem
pytorch-ignite==0.2.1 # High-level library to help with training neural networks in PyTorch
fairseq==0.9.0 # Facebook AI Research Sequence-to-Sequence Toolkit
adabound==0.0.5 # AdaBound optimization algorithm, build on PyTorch.
pyro-ppl==1.1.0 # Deep universal probabilistic programming with Python and PyTorch 
torch-sparse==0.4.3 # PyTorch Extension Library of Optimized Autograd Sparse Matrix Opjaxerations
torch-scatter==1.4.0 # PyTorch Extension Library of Optimized Scatter Operations
gpytorch==0.3.6 # An implementation of Gaussian Processes in Pytorch
pytorch-nlp==0.5.0 # Text utilities and datasets for PyTorch
pytorch-lightning==0.5.3.2 # The Keras for ML researchers using PyTorch
speedtorch==0.1.5 # Fast Pinned CPU -> GPU transfer
catalyst==19.12.1 # Catalyst. High-level utils for PyTorch DL & RL research.

## Sklearn Ecosystem:
skorch==0.7.0 # A scikit-learn compatible neural network library that wraps pytorch
scikit-multilearn==0.2.0 # Scikit-multilearn is a BSD-licensed library for multi-label classification that is built on top of the well-known scikit-learn ecosystem.
sk-dist==0.1.6 # Distributed scikit-learn meta-estimators with PySpark

## Others
jax==0.1.55 # GPU- and TPU-backed NumPy with differentiation and JIT compilation.
jaxlib==0.1.37 # XLA library for JAX - required for jax
onnx==1.6.0 # Open Neural Network Exchange
onnxruntime==1.0.0 # ONNX Runtime Python bindings
nevergrad==0.2.3 # A Python toolbox for performing gradient-free optimization
featuretools==0.13.0 # A framework for automated feature engineering
vecstack==0.4.0 # Python package for stacking (machine learning technique)
mlens==0.2.3 # Machine Learning Ensemble Library
geopy==1.20.0 # Python Geocoding Toolbox
tensorly==0.4.4 # making tensor learning simple and accessible
mmdnn==0.2.5 # Deep learning model converter, visualization and editor.
kmodes==0.10.1 # implementations of the k-modes and k-prototypes clustering algorithms for clustering categorical data.

## Jupyter Ecosystem
jupyter-console==6.0.0 # Jupyter terminal console
jupyter-kernel-gateway==2.4.0 # A web server for spawning and communicating with Jupyter kernels
ipyparallel==6.2.4 # Interactive Parallel Computing with IPython
jupyter_http_over_ws==0.0.6 # Jupyter support for HTTP-over-ws
papermill==1.2.1 # Parametrize and run Jupyter and nteract Notebooks
ipython-sql==0.3.9 # RDBMS access via IPython
ipywidgets==7.5.1 # IPython HTML widgets for Jupyter
widgetsnbextension==3.5.1 # IPython HTML widgets for Jupyter
nbinteract==0.2.4 # Export interactive HTML pages from Jupyter Notebooks
qgrid==1.1.1 # a Jupyter notebook widget which uses SlickGrid to render pandas DataFrames within a Jupyter notebook
ipyaggrid==0.2.0 # Jupyter widget - ag-grid in the notebook
pivottablejs==0.9.0 # PivotTable.js integration for Jupyter/IPython Notebook
tensorwatch==0.8.7 # Interactive Realtime Debugging and Visualization for AI
bqplot==0.12.1 # Plotting library for IPython/Jupyter Notebooks
ipyexperiments==0.1.16 # jupyter/ipython experiment containers for GPU and general RAM re-use and memory leaks detection.
jupyter-server-proxy==1.2.0 # Jupyter server extension to supervise and proxy web services
facets-overview==1.0.0 # Python code to support the Facets Overview visualization
nbdev==0.2.0 # Writing a library entirely in notebooks
nbval==0.9.4 # A py.test plugin to validate Jupyter notebooks
ipysheet==0.4.3 # Spreadsheet in the Jupyter notebook

#
####### Visualization Libs #######
#
## General Visualization Libs
bokeh==1.4.0 # interactive visualization library that targets modern web browsers for presentation - 20MB
plotly==4.4.1 # interactive visualizations - 34MB
holoviews==1.12.7 # Stop plotting your data - annotate your data and let it visualize itself.
chartify==2.7.0 # Python library to make plotting simpler for data scientists
plotly_express==0.4.1 # Plotly Express - a high level wrapper for Plotly.py
pygal==2.4.0 # A python svg graph plotting library
scikit-plot==0.3.7 # An intuitive library to add plotting functionality to scikit-learn objects.

## Vega/Altair Ecosystem
altair==4.0.0 # A declarative statistical visualization library for Python.
vega==2.6.0 # IPyVega: An IPython/Jupyter widgetx for Vega 3 and Vega-Lite 2
vega_datasets==0.8.0 # A Python package for offline access to Vega datasets
pdvega==0.1 # Pandas plotting interface to Vega and Vega-Lite
nx-altair==0.1.4 # Draw NetworkX graphs with Altair

# Dashboard Tools
panel==0.7.0 # A high level dashboarding library for python visualization libraries.
dash==1.7.0 # The core dash backend
dash-html-components==1.0.2 # HTML components
dash-core-components==1.6.0 # Supercharged components
dash-table==4.5.1 # Interactive DataTable component (new!)

## Pandas Visualization Tools
missingno==0.4.2 # Missing data visualization module for Python.
pandas-bokeh==0.4.2 # Bokeh plotting backend for Pandas, GeoPandas & Pyspark

## Training Visualization Tools
tensorboard_logger==0.1.0
visualdl==1.3.0 # Visualize Deep Learning

## ML Model Exploration
yellowbrick==1.0.1 # A suite of visual analysis and diagnostic tools for machine learning.
eli5==0.10.1 # Debug machine learning classifiers and explain their predictions
shap==0.33.0 # A unified approach to explain and visualize the output of any machine learning model.
lime==0.1.1.36 # Local Interpretable Model-Agnostic Explanations for machine learning classifiers
scikit-plot==0.3.7 # An intuitive library to add plotting functionality to scikit-learn objects.
pyldavis==2.1.2 # Interactive topic model visualization
keract==2.8.5 # Keras Activations and Gradients

## Task-specific Visualization Libs
umap-learn==0.3.10 # Uniform Manifold Approximation and Projection
wordcloud==1.6.0 # A little word cloud generator
folium==0.10.1 # Make beautiful maps with Leaflet.js & Python
visdom==0.1.8.9 # A tool for visualizing live, rich data for Torch and Numpy

#
####### Database Libs #######
#
dataset==1.1.2 # Toolkit for Python-based database access.
minio==5.0.5 # S3 client
pymongo==3.10.0 # Mongo Client
db.py==0.5.3 # easier way to interact with your databases, support for  PostgreSQL, MySQL, SQLite, Redshift, MS SQL Server, Oracle
elasticsearch==7.1.0 # Python client for Elasticsearch
neo4j-driver==1.7.6 # Neo4j Bolt driver for Python
lazydata==1.0.19  # Scalable data dependencies
redis==3.3.11 # Python client for Redis key-value store
cx-Oracle==7.3.0 # Python interface to Oracle
kafka-python==1.4.7 # Pure Python client for Apache Kafka
tinydb==3.15.1 # TinyDB is a tiny, document oriented database optimized for your happiness :)
PyMySQL==0.9.3 # Pure Python MySQL Driver
boto3==1.10.39 # The AWS SDK for Python
apache-libcloud==2.7.0 # A standard Python library that abstracts away differences among multiple cloud provider APIs.
google-cloud-storage==1.23.0 # Google Cloud Storage API client library
filedepot==0.7.1 # Toolkit for storing files and attachments in web applications
prometheus_client==0.7.1 # Python client for the Prometheus monitoring system.
cloudant==2.12.0 # Cloudant / CouchDB Client Library
influxdb==5.2.3 # InfluxDB client
pyodbc==4.0.27 # DB API Module for ODBC
pyhdb==0.3.4  # SAP HANA Database Client for Python
lmdb==0.98 # Universal Python binding for the LMDB 'Lightning' Database

#
####### Web Frameworks #######
#
Flask==1.1.1 # A simple framework for building complex web applications.
fastapi==0.45.0 # FastAPI framework, high performance, easy to learn, fast to code, ready for production
starlette==0.12.9 # fastapi requires starlette == 0.12.9 - The little ASGI library that shines.
graphene==2.1.8 # GraphQL Framework for Python
uvicorn==0.10.8 # The lightning-fast ASGI server.
websockets==8.1 # An implementation of the WebSocket Protocol (RFC 6455 & 7692)
gunicorn==20.0.4 # WSGI HTTP Server for UNIX

#
####### Others #######
#
stormssh==0.7.0 # Management commands to ssh config files.
handout==1.1.2 # Turn Python scripts into handouts with Markdown and figures
numexpr==2.7.0 # Fast numerical expression evaluator for NumPy
docker==4.1.0 # A Python library for the Docker Engine API.
safety==1.8.5 # Safety checks your installed dependencies for known security vulnerabilities.
pipdeptree==0.13.2 # Command line utility to show dependency tree of packages

run_jupyter.sh

#!/bin/bash
cd $HOME/miniconda/bin
source activate jupyter
jupyter notebook

setup-env.sh

#!/bin/bash

cd $HOME
# Get Miniconda and make it the main Python interpreter
wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O $HOME/miniconda.sh
bash ~/miniconda.sh -b -p ~/miniconda 
rm ~/miniconda.sh

cd $HOME/miniconda/bin

source activate


# conda update -y -n base -c defaults conda

conda create -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults python=3.6 


conda install -y  -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults \
            "ipython=7.6.1" \
            "numpy=1.16.4" \
            "scikit-learn=0.20.2" \
            "nbformat=4.*.*" \
            jupyterlab \
            psutil \
            nodejs \
            ipyparallel \
            matplotlib \
            "tqdm=4.35.0" \
            pyzmq \
            cython \
            graphviz \
            numpy \
            matplotlib \
            scipy \
            requests \
            urllib3 \
            pandas \
            six \
            future \
            protobuf \
            zlib \
            boost \
            psutil \
            PyYAML \
            python-crontab \
            ipykernel \
            jupyter_contrib_nbextensions
            cmake \
            Pillow \
            tensorflow-base \
            openmpi \
            unixodbc 
conda activate jupyter

jupyter notebook --generate-config
# Download Jupter config from gist
wget -O ${HOME}/.jupyter/jupyter_notebook_config.py https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/jupyter_notebook_config.py


# Pip install section
mkdir -p ${HOME}/DLR_Work/libraries/
wget -O ${HOME}/DLR_Work/libraries/requirements-full.txt https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/requirements-full.txt




$HOME/.conda/envs/jupyter/bin/pip install --no-cache-dir --upgrade -r ${HOME}/DLR_Work/libraries/requirements-full.txt
 

## extensions are comment uncomment, which might be needed
###################################################################
###################################################################
#  # Activate and configure extensions
# jupyter contrib nbextension install --user
# # nbextensions configurator
# jupyter nbextensions_configurator enable --user
# # Active nbresuse
# jupyter serverextension enable --py nbresuse
# # Activate Jupytext
# jupyter nbextension enable --py jupytext
# # Disable Jupyter Server Proxy
# jupyter nbextension disable jupyter_server_proxy/tree
# # Configure nbdime
# nbdime config-git --enable --global
# # Enable useful extensions
# jupyter nbextension enable skip-traceback/main
# # jupyter nbextension enable comment-uncomment/main
# # Do not enable variable inspector: causes trouble: https://github.com/ml-tooling/ml-workspace/issues/10
# # jupyter nbextension enable varInspector/main
# #jupyter nbextension enable spellchecker/main
# jupyter nbextension enable toc2/main
# jupyter nbextension enable execute_time/ExecuteTime
# jupyter nbextension enable collapsible_headings/main
# jupyter nbextension enable codefolding/main
# # Activate Jupyter Tensorboard
# jupyter tensorboard enable
# # Edit notebook config
# # comment for now ### cat $HOME/.jupyter/nbconfig/notebook.json | jq '.toc2={"moveMenuLeft": false}' > tmp.$$.json && mv tmp.$$.json $HOME/.jupyter/nbconfig/notebook.json
# # Activate qgrid
# jupyter nbextension enable --py --sys-prefix qgrid
# # Activate Colab support
# jupyter serverextension enable --py jupyter_http_over_ws
# # Activate Voila Rendering 
# # currently not working jupyter serverextension enable voila --sys-prefix
# # Enable ipclusters
# # comment for now ### ipcluster nbextension enable
# 
# 
# 
# # install jupyterlab
# 
# # Required for jupytext and matplotlib plugins
# jupyter lab build
# # jupyterlab installed in requirements section
# jupyter labextension install @jupyter-widgets/jupyterlab-manager
# 
# jupyter labextension install @jupyterlab/toc
# jupyter labextension install jupyterlab_tensorboard
# # install jupyterlab git
# jupyter labextension install @jupyterlab/git
# pip install jupyterlab-git 
# jupyter serverextension enable --py jupyterlab_git
# # For Matplotlib: https://github.com/matplotlib/jupyter-matplotlib
# jupyter labextension install jupyter-matplotlib
# # Install jupyterlab language server support
# pip install --pre jupyter-lsp
# jupyter labextension install @krassowski/jupyterlab-lsp
# # For Bokeh
# jupyter labextension install jupyterlab_bokeh
# # For Plotly
# jupyter labextension install @jupyterlab/plotly-extension
# jupyter labextension install jupyterlab-chart-editor
# # For holoview
# jupyter labextension install @pyviz/jupyterlab_pyviz
# # Install jupyterlab_iframe - https://github.com/timkpaine/jupyterlab_iframe
# pip install jupyterlab_iframe
# jupyter labextension install jupyterlab_iframe
# jupyter serverextension enable --py jupyterlab_iframe
# # Install jupyterlab variable inspector - https://github.com/lckr/jupyterlab-variableInspector
# jupyter labextension install @lckr/jupyterlab_variableinspector
# # Install jupyterlab code formattor - https://github.com/ryantam626/jupyterlab_code_formatter
# jupyter labextension install @ryantam626/jupyterlab_code_formatter
# pip install jupyterlab_code_formatter
# jupyter serverextension enable --py jupyterlab_code_formatter
# # Install go-to-definition extension 
# jupyter labextension install @krassowski/jupyterlab_go_to_definition
# # Install ipysheet jupyterlab extension
# jupyter labextension install ipysheet
# # Deprecation and validations:
# # Install jupyterlab_templates - https://github.com/timkpaine/jupyterlab_templates
# # Does not work currently
# # pip install jupyterlab_templates
# # jupyter labextension install jupyterlab_templates
# # jupyter serverextension enable --py jupyterlab_templates
# # Install jupyterlab-data-explorer: https://github.com/jupyterlab/jupyterlab-data-explorer
# # alpha version jupyter labextension install @jupyterlab/dataregistry-extension
# # Install jupyterlab system monitor: https://github.com/jtpio/jupyterlab-system-monitor
# # Activate ipygrid in jupterlab
# # Problems with terminal: jupyter labextension install ipyaggrid
# # Install qgrid
# # Not compatible to jupyterlab 1.x: https://github.com/quantopian/qgrid/issues/261
# # DO not install for now jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor
#     # Install voyagar data grid
# # Does not work with 1.1.1: 
# jupyter labextension install jupyterlab_voyager
# # Too big dependency: https://github.com/InsightSoftwareConsortium/itkwidgets
# # Too Big: Install ipyleaflet
# pip install --no-cache-dir ipyleaflet
# jupyter labextension install jupyter-leaflet
# jupyter lab build
# # Cleanup
# # Clean jupyter lab cache: https://github.com/jupyterlab/jupyterlab/issues/4930
# jupyter lab clean
# jlpm cache clean


###################################################################
###################################################################



source activate jupyter 

pip install akima


mkdir -p ${HOME}/DLR_Work
cd ${HOME}/DLR_Work

wget -O ${HOME}}/DLR_Work/install_jupyter.sh https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/install_jupyter.sh

bash ./install_jupyter.sh  
mkdir -p ${HOME}/DLR_Work/repo
cd ${HOME}/DLR_Work/repo

source activate jupyter

git clone https://gitlab.dlr.de/mans_is/analysis_atmospheric_so2.git
cd analysis_atmospheric_so2
pip install -e .

mkdir -p ${HOME}/DLR_Work/

wget -O ${HOME}/DLR_Work/run_jupyter.sh https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/run_jupyter.sh

bash ${HOME}/DLR_Work/run_jupyter.sh 

To install some package from txt file

while read requirement; do conda install -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults --yes $requirement; done < /users/mans_is/DLR_Work/analysis_atmospheric_so2/requirements_dev.txt

To install pip

$HOME/.conda/envs/jupyter/bin/pip install akima





conda install -y  -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults \
            tqdm \
            pyzmq \
            cython \
            graphviz \
            numpy \
            matplotlib \
            scipy \
            requests \
            urllib3 \
            pandas \
            six \
            future \
            protobuf \
            zlib \
            boost \
            psutil \
            PyYAML \
            python-crontab \
            ipykernel \
            jupyter_contrib_nbextensions
            cmake \
            Pillow \
            tensorflow-base \
            openmpi \
            unixodbc 

# Install pytorch - cpu only
conda install -y -c pytorch "pytorch==1.3.*" torchvision cpuonly

# Faiss - A library for efficient similarity search and clustering of dense vectors. 
conda install -y -c pytorch faiss-cpu

To Install requirements-full.txt

 mkdir -p ${HOME}/DLR_Work/libraries/
 wget -O ${HOME}/DLR_Work/libraries/requirements-full.txt https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/requirements-full.txt


 $HOME/.conda/envs/jupyter/bin/pip install --no-cache-dir --upgrade -r ${HOME}/DLR_Work/libraries/requirements-full.txt

For already installed jupyter notebook:

conda create --name tf_1  -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults --yes --quiet \
    "matplotlib>=3.0.3" \
    "numpy>1.16.7" \
    "scipy>=1.2.1" \
    "pillow>=4.3.0" \
    "scikit-image" \
    python=3.6\
    ipykernel \
    seaborn\
    scikit-learn

source activate tf_1;\
conda --name tf_1 install --yes --quiet -c anaconda -c conda-forge keras "tensorflow-base==1.*.*" "tensorflow-gpu==1.*.*";
rm -r -f /opt/conda/pkgs/
python -m ipykernel install --user --name "tf_1" --display-name "Tensorflow 1 latest";\
source deactivate;

For conda clean installation:
https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/install_jupyter.sh

# Download Jupter install script from the gist

mkdir -p ${HOME}/DLR_Work
cd ${HOME}/DLR_Work

wget -O ${HOME}}/DLR_Work/install_jupyter.sh https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/install_jupyter.sh

bash ./install_jupyter.sh  
mkdir -p ${HOME}/DLR_Work/repo
cd ${HOME}/DLR_Work/repo

conda activate jupyter

git clone https://gitlab.dlr.de/mans_is/analysis_atmospheric_so2.git
cd analysis_atmospheric_so2
pip install -e .

To install only basic jupyer envirmoent without jupyterlab addons:

#!/bin/bash
cd /data/inst/anaconda/bin
source activate

# conda update -y -n base -c defaults conda

conda create -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults python=3.6 


conda install -y  -n jupyter -c rapidsai -c nvidia -c numba -c pytorch -c conda-forge -c defaults \
            "ipython=7.6.1" \         
            "numpy=1.16.4" \
            "scikit-learn=0.20.2" \
            "nbformat=4.*.*" \
            jupyterlab \
            psutil \
            nodejs \
            ipyparallel \
            matplotlib \
            "tqdm=4.35.0" \
            pyzmq \
            cython \
            graphviz \
            numpy \
            matplotlib \
            scipy \
            requests \
            urllib3 \
            pandas \
            six \
            future \
            protobuf \
            zlib \
            boost \
            psutil \
            PyYAML \
            python-crontab \
            ipykernel \
            jupyter_contrib_nbextensions
            cmake \
            Pillow \
            tensorflow-base \
            openmpi \
            unixodbc 
conda activate jupyter

jupyter notebook --generate-config
# Download Jupter config from gist
wget -O ${HOME}/.jupyter/jupyter_notebook_config.py https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/jupyter_notebook_config.py


# Pip install section
mkdir -p ${HOME}/DLR_Work/libraries/
wget -O ${HOME}/DLR_Work/libraries/requirements-full.txt https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/requirements-full.txt




$HOME/.conda/envs/jupyter/bin/pip install --no-cache-dir --upgrade -r ${HOME}/DLR_Work/libraries/requirements-full.txt


mkdir -p ${HOME}/DLR_Work/repo
cd ${HOME}/DLR_Work/repo

conda activate jupyter

git clone https://gitlab.dlr.de/mans_is/analysis_atmospheric_so2.git
cd analysis_atmospheric_so2
pip install -e .

To install miniconda and setup jupyter env:

wget -O ${HOME}/setup-env.sh https://gist.githubusercontent.com/IslamAlam/71146ba7e92c9c263cfd61fb7b8c1e16/raw/setup-env.sh

bash ${HOME}/setup-env.sh