Generate blog post images

dask_blog_image_creation.ipynb

import dask
import dask.array as da
import numpy as np

from dask import delayed, persist, compute
from dask.distributed import Client

client = Client()

## create inputs with a bunch of independent normals
beta = np.random.random(100) # random beta coefficients, no intercept
X = da.random.normal(0, 1, size=(1000000, 100), chunks=(100000, 100))
y = X.dot(beta) + da.random.normal(0, 1, size=1000000, chunks=(100000,))

## make sure all chunks are ~equally sized
client.rebalance([X, y])

for color in ['white', 'black']:
    X.visualize(filename='blog/images/X-%s.svg' % color, 
                node_attr={'color': color}, 
                edge_attr={'color': color}, 
                bgcolor='white',
               pencolor=color)

## quick step-size calculation to guarantee convergence
_, s, _ = da.linalg.svd_compressed(2 * X.T.dot(X), k=1)
step_size = 1 / s.compute() - 1e-8

max_steps = 100
tol = 1e-8
beta_hat = np.zeros(100) # initial guess
X, y = persist(X, y)

for color in ['white', 'black']:
    for k in range(1):
        Xbeta = X.dot(beta_hat)
        func = ((y - Xbeta)**2).sum()
        gradient = 2 * X.T.dot(Xbeta - y)

        ## update
        obeta = beta_hat
        beta_hat = beta_hat - step_size * gradient
        new_func = ((y - X.dot(beta_hat))**2).sum()
        dask.visualize(beta_hat, new_func, filename='blog/images/grad_step-%s.svg' % color, 
                    node_attr={'color': color}, 
                    edge_attr={'color': color}, 
                    bgcolor='transparent',
                      pencolor=color)
        beta_hat, func, new_func = compute(beta_hat, func, new_func) # the only dask