ian-whitestone

Setting Up Jupyter Lab on an EC2

1. Install Jupyter Lab

conda install -c conda-forge jupyterlab

2. Create certs

openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mycert.pem -out mycert.pem

ian-whitestone

# Source: https://towardsdatascience.com/a-data-science-for-good-machine-learning-project-walk-through-in-python-part-one-1977dd701dbc

import pandas as pd

# Number of missing in each column
missing = pd.DataFrame(data.isnull().sum()).rename(columns = {0: 'total'})

# Create a percentage missing
missing['percent'] = missing['total'] / len(data)

ian-whitestone

"""Generate a bunch of fake avro data and upload to s3

Running in python 3.7. Installed the following:

- pip install Faker
- pip install fastavro
- pip install boto3
- pip install graphviz
- brew install graphviz
"""

ian-whitestone

import sys

import dask.bag as db


def gt(x):
    return x > 3

def even(x):
    return x % 2 == 0

ian-whitestone

• https://github.com/ian-whitestone/toronto-housing/blob/master/data/analysis/analysis.R
• https://github.com/ian-whitestone/khp-analytics/blob/master/analysis/eda.R
• https://github.com/ian-whitestone/nba-dfs/blob/master/NBA_eda_report.Rmd

ian-whitestone

# User defined constants
HOME="/home/ianwhitestone/"
LOGS_DIR="logs"
MASTER_LOG_FILENAME="master.txt"
SECRETS_FILE="$HOME/.secrets"

# Input Arguments
MODULE=$1
FUNCTION=$2

ian-whitestone

"""
Trigger slack notifications
"""
import argparse
import logging
import os

from slack.web.client import WebClient

LOGGER = logging.getLogger(__name__)

ian-whitestone

import pickle

PROXY_URL = 'https://z1h4spb3u7.execute-api.us-west-1.amazonaws.com/proxy_us_west_1'

def proxy_request(url):
    proxy_response = requests.post(
        PROXY_URL,
        data={'url': url}
    )
    if not proxy_response.ok:

ian-whitestone

// https://github.com/mui-org/material-ui/tree/master/examples/create-react-app
import React from 'react';
import Container from '@material-ui/core/Container';
import Typography from '@material-ui/core/Typography';
import Box from '@material-ui/core/Box';
import { makeStyles } from '@material-ui/core/styles';
import Link from '@material-ui/core/Link';
import SvgIcon from '@material-ui/core/SvgIcon';
import Button from '@material-ui/core/Button';

ian-whitestone

"""
A simple progress bar that just shows the total number of "things" processed
and the time elapsed
"""
import time
from datetime import timedelta

from rich.progress import Progress, ProgressColumn, Task, TaskID, Text, TextColumn