azure cluster

fix_gdal_py2.sh

#!/usr/bin/env bash

/usr/bin/anaconda/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/36e12d84a418d2274ef79b1d853073cc200535cc/r.txt
#/usr/bin/anaconda/bin/conda update --all
/usr/bin/anaconda/bin/conda install hdf4

i.sh

#!/usr/bin/env bash

/usr/bin/anaconda/envs/py35/bin/pip install pyproj

i2.sh

#!/usr/bin/env bash

/usr/bin/anaconda/envs/py35/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/76a1048713b28dc9b8e49a6956e7368cc16ca58c/r.txt

i3.sh

#!/usr/bin/env bash

/usr/bin/anaconda/envs/py35/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/36e12d84a418d2274ef79b1d853073cc200535cc/r.txt

i4.sh

#!/usr/bin/env bash

/usr/bin/anaconda/bin/conda install -c conda-forge osmnx

i5.sh

#!/usr/bin/env bash

while read requirement; do /usr/bin/anaconda/bin/conda install --yes $requirement || /usr/bin/anaconda/envs/py35/bin/pip install $requirement; done < hhttps://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/ea2af6f86e8f1c54adbb08a86946a78e63316bbc/r2.txt

i6.sh

#!/usr/bin/env bash

/usr/bin/anaconda/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/36e12d84a418d2274ef79b1d853073cc200535cc/r.txt

install.sh

#!/usr/bin/env bash

#/usr/bin/anaconda/bin/conda install --yes --file https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/9ffd29fff59a20b00f604a3aa3a06d0a972541e9/r.txt


while read requirement; do /usr/bin/anaconda/bin/conda install --yes $requirement || /usr/bin/anaconda/envs/py35/bin/pip install $requirement; done < https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/9ffd29fff59a20b00f604a3aa3a06d0a972541e9/r.txt

install_py3_bothnode.txt

#!/usr/bin/env bash
/usr/bin/anaconda/envs/py35/bin/conda config --add channels conda-forge
/usr/bin/anaconda/envs/py35/bin/conda install -n py35 rtree --yes
/usr/bin/anaconda/envs/py35/bin/conda install -n py35 osmnx --yes
#/usr/bin/anaconda/envs/py35/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/d2b932e42ce2226b0b4794155c6d030219e420b0/r.txt

py2pip.sh

#!/usr/bin/env bash

/usr/bin/anaconda/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/1152a10fa8101dbfd1aecd5a9f9994337aa166c8/r.txt

py3pip.sh

#!/usr/bin/env bash

/usr/bin/anaconda/envs/py35/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/1152a10fa8101dbfd1aecd5a9f9994337aa166c8/r.txt

r.txt

osmnx
pymc3==3.1
pyproj==1.9.5.1
bokeh==0.12.9
boto3==1.4.4
botocore==1.7.47
Bottleneck==1.2.0
branca==0.2.0
carto==1.1.1
cartoframes
geopandas
shapely

r2.txt

dask==0.15.4
folium==0.3.0
GDAL==2.2.0
geocoder==1.23.2

requirement.txt

adal==0.4.7
anaconda-client==1.6.3
appdirs==1.4.3
appnope==0.1.0
asn1crypto==0.23.0
awscli==1.11.189
azure==2.0.0
azure-batch==3.0.0
azure-common==1.1.8
azure-datalake-store==0.0.17
azure-graphrbac==0.30.0
azure-keyvault==0.3.7
azure-mgmt==1.0.0
azure-mgmt-authorization==0.30.0
azure-mgmt-batch==4.0.0
azure-mgmt-cdn==0.30.3
azure-mgmt-cognitiveservices==1.0.0
azure-mgmt-compute==1.0.0
azure-mgmt-containerregistry==0.2.1
azure-mgmt-datalake-analytics==0.1.6
azure-mgmt-datalake-nspkg==2.0.0
azure-mgmt-datalake-store==0.1.6
azure-mgmt-devtestlabs==2.0.0
azure-mgmt-dns==1.0.1
azure-mgmt-documentdb==0.1.3
azure-mgmt-iothub==0.2.2
azure-mgmt-keyvault==0.31.0
azure-mgmt-logic==2.1.0
azure-mgmt-monitor==0.2.1
azure-mgmt-network==1.0.0
azure-mgmt-nspkg==2.0.0
azure-mgmt-rdbms==0.1.0
azure-mgmt-redis==4.1.1
azure-mgmt-resource==1.1.0
azure-mgmt-scheduler==1.1.3
azure-mgmt-sql==0.5.3
azure-mgmt-storage==1.0.0
azure-mgmt-trafficmanager==0.30.0
azure-mgmt-web==0.32.0
azure-nspkg==2.0.0
azure-servicebus==0.21.1
azure-servicefabric==5.6.130
azure-servicemanagement-legacy==0.20.6
azure-storage==0.34.3
azure-storage-blob==0.37.1
azure-storage-common==0.37.1
azure-storage-nspkg==2.0.0
beautifulsoup4==4.5.3
bkcharts==0.2
bleach==2.0.0
bokeh==0.12.9
boto3==1.4.4
botocore==1.7.47
Bottleneck==1.2.0
branca==0.2.0
carto==1.1.1
cartoframes==0.3.0b5
certifi==2017.7.27.1
cffi==1.11.2
chardet==3.0.4
click==6.7
click-plugins==1.0.3
cligj==0.4.0
cloudpickle==0.3.1
clyent==1.2.1
colorama==0.3.7
colour==0.1.4
cryptography==2.1.3
cycler==0.10.0
Cython==0.25.2
dask==0.15.4
decorator==4.1.2
descartes==1.1.0
distributed==1.17.1
docutils==0.14
dweepy==0.3.0
entrypoints==0.2.3
fastparquet==0.1.3
Fiona==1.7.7
Flask==0.12
folium==0.3.0
future==0.15.2
futures==3.1.1
GDAL==2.2.0
geocoder==1.23.2
geopandas==0.2.1
geopy==1.11.0
geos==0.2.0
graphviz==0.8.1
h5py==2.7.1
hdbscan==0.8.10
heapdict==1.0.0
html5lib==0.999999999
idna==2.6
imageio==2.1.2
imutils==0.4.2
ipykernel==4.6.1
ipython==6.2.1
ipython-genutils==0.2.0
ipywidgets==6.0.0
isodate==0.6.0
itsdangerous==0.24
jedi==0.11.0
Jinja2==2.9.6
jmespath==0.9.3
joblib==0.11
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.0.1
jupyter-console==5.1.0
jupyter-core==4.3.0
keyring==10.5.0
llvmlite==0.20.0
locket==0.2.0
lxml==3.7.3
Mako==1.0.7
MarkupSafe==1.0
matplotlib==2.0.2
mistune==0.7.4
moviepy==0.2.3.2
msgpack-python==0.4.8
msrest==0.4.19
msrestazure==0.4.16
munch==2.1.1
nbconvert==5.2.1
nbformat==4.3.0
networkx==1.11
notebook==5.0.0
numba==0.35.0
numexpr==2.6.2
numpy==1.13.3
oauthlib==2.0.6
olefile==0.44
opencv-python==3.2.0.7
osmnx==0.5.1
pandas==0.21.0
pandocfilters==1.4.1
parso==0.1.0
partd==0.3.8
patsy==0.4.1
pexpect==4.2.1
pickleshare==0.7.4
Pillow==4.1.1
prompt-toolkit==1.0.15
psutil==5.2.1
psycopg2==2.7.1
ptyprocess==0.5.1
pyarrow==0.7.1
pyasn1==0.3.7
pycparser==2.18
pycurl==7.43.0
Pygments==2.2.0
pygpu==0.7.1
PyJWT==1.5.3
pykalman==0.9.5
pymc3==3.1
pyparsing==2.2.0
pyproj==1.9.5.1
pyrestcli==0.6.4
PySAL==1.14.2
pystan==2.16.0.0
python-dateutil==2.6.1
python-snappy==0.5.1
pytz==2017.3
PyWavelets==0.5.2
PyYAML==3.12
pyzmq==16.0.2
qtconsole==4.3.0
ratelim==0.1.6
requests==2.18.4
requests-oauthlib==0.8.0
rsa==3.4.2
Rtree==0.8.3
s3fs==0.1.2
s3transfer==0.1.11
scikit-image==0.13.0
scikit-learn==0.18.1
scipy==0.19.1
seaborn==0.7.1
Shapely==1.5.17.post1
simplegeneric==0.8.1
six==1.11.0
sortedcontainers==1.5.3
SQLAlchemy==1.1.5
statsmodels==0.8.0
tables==3.4.2
tblib==1.3.2
terminado==0.6
testpath==0.3
Theano==0.9.0
thrift==0.10.0
toolz==0.8.2
tornado==4.5.2
tqdm==4.19.4
traitlets==4.3.2
urllib3==1.22
vincent==0.4.4
wcwidth==0.1.7
webcolors==1.7
webencodings==0.5.1
Werkzeug==0.11.10
widgetsnbextension==2.0.0
xmltodict==0.11.0
zict==0.1.2

requirement_gps.txt

geopy
numpy
hashlib
carto==1.1.1
cartoframes==0.3.0b5
geopandas
shapely
rtree
osmnx==0.5.1
pyproj==1.9.5.1
bokeh==0.12.9
boto3==1.4.4

rtree_conda2_install.sh

#!/usr/bin/env bash

/usr/bin/anaconda/bin/conda config --add channels conda-forge
/usr/bin/anaconda/bin/conda install rtree --yes

spin_up_new_cluster.sh

#!/usr/bin/env bash
/usr/bin/anaconda/envs/py35/bin/conda config --add channels conda-forge

/usr/bin/anaconda/envs/py35/bin/conda install -n py35 rtree --yes
/usr/bin/anaconda/envs/py35/bin/conda install -n py35 osmnx --yes
/usr/bin/anaconda/envs/py35/bin/conda install -n py35 libgdal --yes
/usr/bin/anaconda/envs/py35/bin/conda install -n py35 gdal --yes

/usr/bin/anaconda/envs/py35/bin/conda install -n root rtree --yes
/usr/bin/anaconda/envs/py35/bin/conda install -n root libgdal --yes
/usr/bin/anaconda/envs/py35/bin/conda install -n root gdal --yes

/usr/bin/anaconda/envs/py35/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/4bd606b207446cdb9db3f869bcc4b40c2184c39e/r.txt
/usr/bin/anaconda/bin/pip install -r https://gist.githubusercontent.com/djfan/ccdfac26415196021adacb2d52455c60/raw/4bd606b207446cdb9db3f869bcc4b40c2184c39e/r.txt

• requirement_gps.txt
• script action: pyspark2
  • /usr/bin/anaconda/bin/pip
  • /usr/bin/anaconda/bin/conda
• script action: pyspark3
  • /usr/bin/anaconda/envs/py35/bin/pip
  • /usr/bin/anaconda/envs/py35/bin/conda
• test %%local using py3
• template to spin up a new cluster
• template to add new packages through script action