joshuacook

eval "$(
    git for-each-ref --shell --format \
    "git --no-pager log -1 --date=iso --format='%%ad '%(align:left,25)%(refname:short)%(end)' %%h %%s' \$(git merge-base %(refname:short) main);" \
    refs/heads
)" | sort

joshuacook

Local Spark Development

Infrastructure

Launch a Jupyter Notebook server using Docker and the jupyter/pyspark-notebook image on your local machine.

Copy and paste the below into your terminal.

docker run -d -v `pwd`:/home/jovyan -p 80:8888 jupyter/pyspark-notebook

joshuacook

joshuacook

while true; do docker stats --format "{{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}" >> stats.txt; done

joshuacook

1. Make a new cloud instance with ports 2181 and 9092 open.
2. Install Docker and Docker-Compose

   $ curl -sSL https://get.docker.com | sh
   $ sudo curl -L https://github.com/docker/compose/releases/download/1.23.1/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
   $ sudo chmod +x /usr/local/bin/docker-compose
   

3. Clone this repo:

$ git clone https://github.com/wurstmeister/kafka-docker

joshuacook

import os

files = [f.split(' - ') for f in os.listdir('./') if 'ipynb' not in f]

files_up = [[v.zfill(3), n] for v, n in files]

files = [v+" - "+n for v, n in files]
files_up = [v+" - "+n for v, n in files_up]
change = list(zip(files, files_up))

joshuacook

\begin{displaymath}
   f(x) = \left\{
     \begin{array}{lr}
       1 & : x \in \mathbb{Q}\\
       0 & : x \notin \mathbb{Q}
     \end{array}
   \right.
\end{displaymath}