ahmaurya

<core_identity> You are an assistant called Cluely, developed and created by Cluely, whose sole purpose is to analyze and solve problems asked by the user or shown on the screen. Your responses must be specific, accurate, and actionable. </core_identity>

<general_guidelines>

• NEVER use meta-phrases (e.g., "let me help you", "I can see that").
• NEVER summarize unless explicitly requested.
• NEVER provide unsolicited advice.
• NEVER refer to "screenshot" or "image" - refer to it as "the screen" if needed.
• ALWAYS be specific, detailed, and accurate.

ahmaurya

from IPython.display import HTML

HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }

ahmaurya

\usepackage{xcolor}
\usepackage{listings}

\definecolor{maroon}{cmyk}{0, 0.87, 0.68, 0.32}
\definecolor{halfgray}{gray}{0.55}
\definecolor{ipython_frame}{RGB}{207, 207, 207}
\definecolor{ipython_bg}{RGB}{247, 247, 247}
\definecolor{ipython_red}{RGB}{186, 33, 33}
\definecolor{ipython_green}{RGB}{0, 128, 0}
\definecolor{ipython_cyan}{RGB}{64, 128, 128}

ahmaurya

# set up logging
logging.basicConfig(level=logging.INFO,
    format='%(asctime)s %(name)s %(levelname)s %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
    filename='log.txt',
    filemode='w')
self.logger = logging.getLogger('LogDisplayName')

# setting timezone for correct reading of string datetimestamps
os.environ['TZ'] = 'GMT'

ahmaurya

import operator as op
from functools import reduce
from decimal import Decimal

def FisherSens(totalN, treatedN, totalSuccesses, treatedSuccesses, Gammas):
	## 'FisherSens' 
	## Python code by Abhinav Maurya
	## Original code in R by Devin Caughey at https://rdrr.io/cran/rbounds/src/R/FisherSens.R
	##
	## This function performs a sensitivity analysis for Fisher's Exact Test

ahmaurya

% document setup
\documentclass[twoside,11pt,english,utf8ttf]{article}
\documentclass[twoside,11pt,english,utf8ttf,letterpaper]{article}
\documentclass[twoside,11pt,english,utf8ttf,a4paper]{article}

% packages
\usepackage[1-25]{pagesel}
\usepackage[bf]{caption}
\usepackage[margin=1in]{geometry}
\usepackage[table]{xcolor}

ahmaurya

# %sh -e

# shell commands to install mpl_toolkits
sudo pip install matplotlib
cd /databricks
mkdir -p mpl_toolkit
cd mpl_toolkit

wget https://www.andrew.cmu.edu/user/amaurya/docs/95869/basemap-1.0.7.tar.gz
tar -xvf basemap-1.0.7.tar.gz

ahmaurya

# Step I: Start an AWS EMR cluster. Choose Spark in the software to be installed and provide s3://elasticmapreduce.bootstrapactions/ipython-notebook/install-ipython-notebook as the bootstrap action

# Step II: Login onto the master node of the cluster and execute "sudo pip install jupyter databricks_test_helper matplotlib seaborn"

# Step III: Execute the command "jupyter notebook list" which should show an IPython notebook URL which ends in something like "?token=e52347f8d93d122519e4b9d8df7e34b38d7074f76539e225". Copy just its token which should look something like "e52347f8d93d122519e4b9d8df7e34b38d7074f76539e225". This will help you login on the notebook server.

# Step IV: Execute the command: "ssh -o ServerAliveInterval=10 -i amazon-key-pair.pem -N -L 8192:ec2-52-86-172-186.compute-1.amazonaws.com:8192 hadoop@ec2-52-86-172-186.compute-1.amazonaws.com" on your local machine such as your laptop. This command opens an ssh tunnel from your machine to the AWS cluster master on port 8192. amazon-key-pair.pem i

ahmaurya

import os
import sys
import os.path
 
spark_home = os.environ.get('SPARK_HOME', None)
if not spark_home:
      raise ValueError('SPARK_HOME environment variable is not set')
sys.path.insert(0, os.path.join(spark_home, 'python'))
sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-src.zip'))
if 'sc' not in vars() and 'sc' not in globals():

ahmaurya

# improved list of objects
.ls.objects <- function (pos = 1, pattern, order.by,
                        decreasing=FALSE, head=FALSE, n=5) {
    napply <- function(names, fn) sapply(names, function(x)
                                         fn(get(x, pos = pos)))
    names <- ls(pos = pos, pattern = pattern)
    obj.class <- napply(names, function(x) as.character(class(x))[1])
    obj.mode <- napply(names, mode)
    obj.type <- ifelse(is.na(obj.class), obj.mode, obj.class)
    obj.prettysize <- napply(names, function(x) {