GenevieveBuckley

So, you want to make a new high level graph layer class?

Step 1:

It's common to want to convert something Dask already does, and convert it to use high level graph under the hood.

First you need to find the place in the code where the dask task dictionary is created. Typically this looks like a variable called dsk or dsk_out that is a dictionary mapping the key names to individual tasks.

Found it? Great, this is the spot we're going to insert an instance of your (new, not yet created) high level graph class, eg: dsk_out = MyNewLayer(input_args, ...)

GenevieveBuckley

GenevieveBuckley

# Proposed slices_from_chunks_overlap function 
# Mofified from slices_from_chunks from dask.array.core
from itertools import product
from dask.array.slicing import cached_cumsum


def slices_from_chunks_overlap(chunks, array_shape, depth=1):
    """Translate chunks tuple to a set of slices in product order
    Parameters
    ----------

GenevieveBuckley

# This file was automatically generated by SWIG (http://www.swig.org).
# Version 4.0.2
#
# Do not make changes to this file unless you know what you are doing--modify
# the SWIG interface file instead.


import os
import six
import collections

GenevieveBuckley

#!/bin/bash

# Fernando's nuke and reinstall conda script
# https://twitter.com/fperez_org/status/1447996737063317512

# Download and install miniconda
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O ~/Downloads/miniconda.sh
bash ~/Downloads/miniconda.sh -b -p $HOME/local/conda

conda config --add channels conda-forge

GenevieveBuckley

def match_key(key, keylist):
    # key to match
    key = list(key)    
    key[0] = key[0].rsplit('-', 1)[0]  # remove token
    # search for matches
    for k in keylist:
        temp = list(k)    
        temp[0] = temp[0].rsplit('-', 1)[0]  # remove token
        if temp == key:
            return tuple(k)

GenevieveBuckley

GenevieveBuckley

import dask
import dask.array as da
import numpy as np


def _inner_axes(a_ndim, b_ndim, axes):
    """Given tensordot axes argument, return list of axes to sum over."""
    if isinstance(axes, (int, float)):
        if axes == 0:
            inner_axes_a = []

GenevieveBuckley

import numpy as np
import pandas as pd


def _find_slices(x):
    """An alternative to scipy.ndi.find_objects"""
    unique_vals = np.unique(x)
    unique_vals = unique_vals[unique_vals != 0]
    result = {}
    for val in unique_vals:

GenevieveBuckley

def _tensordot_shape_output(a, b, axes):
    if isinstance(axes, (int, float)):
        if axes == 0:
            shape_out = a.shape + b.shape
            chunks_out = a.chunks + b.chunks
        elif axes > 0:
            shape_out = a.shape[:axes-1] + b.shape[axes-1:]
            chunks_out = a.chunks[:axes-1] + b.chunks[axes-1:]
    else:
        axes_a, axes_b = axes