AlexanderNenninger

Serialize and Deserialize Python Objects From Different Modules While Never Importing Both at the Same Time

Why?

I had a Kubeflow pipline where module size was limited. One can either revert to type unsafe data conversion or depend on a third module for data schemas.

How?

We define functionality and data in separate Python modules. While functionality gets split into different modules, data definitions are placed in a common module. The size of code that needs to be sent to pipeline workers. If data_models becomes to big, e.g. if there are more classes that are can be converted into each other, it can be split up into foo_data_model and bar_data_model.

AlexanderNenninger

*.parquet binary merge=ours
*.parquet filter=lfs diff=lfs merge=lfs -text

AlexanderNenninger

# Lazy Array implementation. The explicit typing allows for static compilation leading to high performance.

using LinearAlgebra
using BenchmarkTools

# Quick implementation of a lazy Array. Yes, it's really *that* simple.
mutable struct LazyFunctionArray{F<:Function,T,N} <: AbstractArray{T,N}
    const f::F                   # Julia 1.8 const field syntax for convenience
    const size::NTuple{N,Int}
    ncalls::Int

AlexanderNenninger

import numpy as np
from numpy import ndarray
import matplotlib.pyplot as plt

from functools import reduce


def up(index):
	i, j = index
	return i - 1, j

AlexanderNenninger

from numba.decorators import jit
import numpy as np

#The original version is here: https://gist.github.com/ximeg/587011a65d05f067a29ce9c22894d1d2
#I made small changes and used numba to do it faster.
  
@jit
def thresholding_algo2(y, lag, threshold, influence):
    signals = np.zeros(len(y))
    filteredY = np.array(y)

AlexanderNenninger

import time
import numba as nb
import numpy as np


def timeit(method):
    "Decorator to time a function"

    def timed(*args, **kw):
        ts = time.time()

AlexanderNenninger

stmt_index = 'ls = [0]*10000\nfor i in range(10000):\n    ls[i] = i\n'
stmt_append = 'ls = []\nfor i in range(10000):\n    ls.append(i)\n'
stmt_comprehension = '[i for i in range(10000)]'

t_index = timeit(stmt_index, number=10000)
# >>> 3.06s

t_append = timeit(stmt_append, number=10000)
# >>> 4.23s

AlexanderNenninger

use std::boxed::Box;

type Callback<U, T> = Box<dyn Fn(&mut U, &T)>;
type Condition<U, T> = Box<dyn Fn(&U, &T) -> bool>;
type Effect<U, T> = Box<dyn Fn(&mut U, &T)>;

fn make_callback<U: 'static, T: 'static>(
    condition: Condition<U, T>,
    effect: Effect<U, T>,
) -> Callback<U, T> {

AlexanderNenninger

use std::boxed::Box;

type Callback = Box<dyn Fn(&mut f64, &f64)>;
type Condition = Box<dyn Fn(&f64, &f64) -> bool>;
type Effect = Box<dyn Fn(&mut f64, &f64)>;

fn step(u: &mut f64, t: &mut f64, callback: &Callback) {
    callback(u, t);
    *t += 1.;
    *u += 1.;

AlexanderNenninger

use std::collections::HashSet;
use std::error::Error;
use std::fmt;
use std::isize;
use std::num::ParseIntError;
use std::str;

pub const INPUT: &str = "input/day13.txt";

#[derive(Debug)]