ahmaurya

from IPython.display import HTML

HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }

ahmaurya

\usepackage{xcolor}
\usepackage{listings}

\definecolor{maroon}{cmyk}{0, 0.87, 0.68, 0.32}
\definecolor{halfgray}{gray}{0.55}
\definecolor{ipython_frame}{RGB}{207, 207, 207}
\definecolor{ipython_bg}{RGB}{247, 247, 247}
\definecolor{ipython_red}{RGB}{186, 33, 33}
\definecolor{ipython_green}{RGB}{0, 128, 0}
\definecolor{ipython_cyan}{RGB}{64, 128, 128}

ahmaurya

# set up logging
logging.basicConfig(level=logging.INFO,
    format='%(asctime)s %(name)s %(levelname)s %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
    filename='log.txt',
    filemode='w')
self.logger = logging.getLogger('LogDisplayName')

# setting timezone for correct reading of string datetimestamps
os.environ['TZ'] = 'GMT'

ahmaurya

import operator as op
from functools import reduce
from decimal import Decimal

def FisherSens(totalN, treatedN, totalSuccesses, treatedSuccesses, Gammas):
	## 'FisherSens' 
	## Python code by Abhinav Maurya
	## Original code in R by Devin Caughey at https://rdrr.io/cran/rbounds/src/R/FisherSens.R
	##
	## This function performs a sensitivity analysis for Fisher's Exact Test

ahmaurya

% document setup
\documentclass[twoside,11pt,english,utf8ttf]{article}
\documentclass[twoside,11pt,english,utf8ttf,letterpaper]{article}
\documentclass[twoside,11pt,english,utf8ttf,a4paper]{article}

% packages
\usepackage[1-25]{pagesel}
\usepackage[bf]{caption}
\usepackage[margin=1in]{geometry}
\usepackage[table]{xcolor}

ahmaurya

# %sh -e

# shell commands to install mpl_toolkits
sudo pip install matplotlib
cd /databricks
mkdir -p mpl_toolkit
cd mpl_toolkit

wget https://www.andrew.cmu.edu/user/amaurya/docs/95869/basemap-1.0.7.tar.gz
tar -xvf basemap-1.0.7.tar.gz

ahmaurya

# Step I: Start an AWS EMR cluster. Choose Spark in the software to be installed and provide s3://elasticmapreduce.bootstrapactions/ipython-notebook/install-ipython-notebook as the bootstrap action

# Step II: Login onto the master node of the cluster and execute "sudo pip install jupyter databricks_test_helper matplotlib seaborn"

# Step III: Execute the command "jupyter notebook list" which should show an IPython notebook URL which ends in something like "?token=e52347f8d93d122519e4b9d8df7e34b38d7074f76539e225". Copy just its token which should look something like "e52347f8d93d122519e4b9d8df7e34b38d7074f76539e225". This will help you login on the notebook server.

# Step IV: Execute the command: "ssh -o ServerAliveInterval=10 -i amazon-key-pair.pem -N -L 8192:ec2-52-86-172-186.compute-1.amazonaws.com:8192 hadoop@ec2-52-86-172-186.compute-1.amazonaws.com" on your local machine such as your laptop. This command opens an ssh tunnel from your machine to the AWS cluster master on port 8192. amazon-key-pair.pem i

ahmaurya

import os
import sys
import os.path
 
spark_home = os.environ.get('SPARK_HOME', None)
if not spark_home:
      raise ValueError('SPARK_HOME environment variable is not set')
sys.path.insert(0, os.path.join(spark_home, 'python'))
sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-src.zip'))
if 'sc' not in vars() and 'sc' not in globals():

ahmaurya

# improved list of objects
.ls.objects <- function (pos = 1, pattern, order.by,
                        decreasing=FALSE, head=FALSE, n=5) {
    napply <- function(names, fn) sapply(names, function(x)
                                         fn(get(x, pos = pos)))
    names <- ls(pos = pos, pattern = pattern)
    obj.class <- napply(names, function(x) as.character(class(x))[1])
    obj.mode <- napply(names, mode)
    obj.type <- ifelse(is.na(obj.class), obj.mode, obj.class)
    obj.prettysize <- napply(names, function(x) {

ahmaurya

library('googleVis')

dd <- read.csv(header = TRUE, text = "Birth,Death,Poet,City
               1797,1869,Mirza Ghalib,Delhi
               1722,1810,Mir Taqi Mir,Lucknow
               1831,1905,Daag Dehlvi,Delhi
               1911,1984,Faiz Ahmed Faiz,Lahore
               1800,1851,Momin Khan Momin,Delhi
               1877,1938,Muhammad Iqbal,Lahore
               1918,2002,Kaifi Azmi,Mumbai