| from datetime import datetime | |
| start_time = datetime.now() | |
| import os | |
| # Open data portal constants | |
| OPENDATA_URL_CASOS_POSITIVOS = 'https://www.datosabiertos.gob.pe/dataset/casos-positivos-por-covid-19-ministerio-de-salud-minsa' | |
| ALTERNATE_URL_CASOS_POSITIVOS = 'https://cloud.minsa.gob.pe/s/Y8w3wHsEdYQSZRp/download' # alternate URL from MINSA cloud service | |
| DESTINATION_CASOS_POSITIVOS_RAW = './data/casos-positivos-raw.csv' | |
| CASOS_POSITIVOS_CLEAN = './data/casos-positivos-utf8.csv' |
| GOOGLE_SPREADSHEET_ID = '<ID_OF_YOUR_SPREADSHEET>' #e.g. 1klG4ihJRyesiWAI_Fq1mv-fErbXtXP4IoVmrJ9hij-k | |
| from google.colab import auth | |
| auth.authenticate_user() | |
| import gspread | |
| from oauth2client.client import GoogleCredentials | |
| gc = gspread.authorize(GoogleCredentials.get_application_default()) |
To run this on EMR, do something akin to the following (with buckets changed as appropriate):
aws emr create-cluster \
--name "NED Conversion" \
--log-uri s3://ned-13arcsec.openterrain.org/logs/ \
--release-label emr-4.0.0 \
--use-default-roles \
--auto-terminate \
--ec2-attributes KeyName=stamen-keypair \
| #!/bin/sh | |
| setup_brew () { | |
| if ![-f "/usr/local/bin/brew"]; then | |
| /usr/bin/ruby -e "$(/usr/bin/curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" | |
| fi | |
| } | |
| setup_ipython () { | |
| brew install readline |
| ... | |
| url(r'^v1/tiles/(?P<tile_user>([^/]+))/(?P<tile_layer>([^/]+))/(?P<tile_zoom>(\d+))/(?P<tile_column>(\d+))/(?P<tile_row>(\d+))\.(?P<tile_format>([a-z]+))$', TileManager.as_view(), name='tile_manager'), | |
| ... |
| brew remove pyqt | |
| brew install pyqt #verify everything works fine | |
| export PYTHONPATH=/usr/local/lib/python2.7/site-packages:$PYTHONPATH | |
| brew tap homebrew/science | |
| brew install qgis --with-grass --with-postgis |
| from fabric.api import * | |
| from fabric.contrib.console import confirm | |
| import tempfile | |
| import re | |
| PROJ_VER='4.8.0' | |
| GEOS_VER='3.3.3' | |
| GDAL_VER='1.9.0' | |
| PROJ_PATH='/usr/local/proj/' + PROJ_VER |