Skip to content

Instantly share code, notes, and snippets.

Avatar
💌
waylonwalker.com/newsletter

Waylon Walker WaylonWalker

💌
waylonwalker.com/newsletter
View GitHub Profile
@WaylonWalker
WaylonWalker / dynamic_pydantic_models.py
Last active March 15, 2023 17:38
The idea is to make pluggable pydantic models where other libs can have config within a libraries config, and be type checked.
View dynamic_pydantic_models.py
from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel, create_model, validator
import toml
class User(BaseModel):
id: int
name = "John Doe"
@WaylonWalker
WaylonWalker / first_nodes.py
Created September 12, 2021 18:56
Write Your First Pipeline Node
View first_nodes.py
"""
first_nodes.py
used in video https://youtu.be/-gEwU-MrPuA
Full Post 👉 https://waylonwalker.com/kedro-your-first-nodes/
"""
from kedro.pipeline import node
View ipython_prompt.py
from IPython.terminal.prompts import Prompts, Token
from pathlib import Path
import os
from platform import python_version
import subprocess
def get_branch():
try:
return (
subprocess.check_output(
View standard_config.py
"""Standard Config.
A module to load tooling config from a users project space.
One day this may become a package, for now its a gist.
Inspired from frustrations that some tools have a tool.ini, .tool.ini,
setup.cfg, or pyproject.toml. Some allow for global configs, some don't. Some
properly follow the users home directory, others end up in a weird temp
directory. Windows home directory is only more confusing. Some will even
respect the users `$XDG_HOME` directory.
View copy_easyclip_to_windows.py
import re
import pyperclip
def copy():
with open('C:/Debian/rootfs/home/<user>/.easyclip') as f:
easyclip = f.readlines(1)
first = re.sub(r"^.*text': '", "", easyclip[0])
last = re.sub(r"'}\n$", "", first)
View KedroBasics_startup.sh
#!/bin/bash
# curl -s -L https://gist.githubusercontent.com/WaylonWalker/0532e9e9f41432d9a375490235286cd6/raw/4bc499518889aa9ad8ee0e410bc44f96a860f1e0/KedroBasics_startup.sh | bash
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get upgrade -yq
apt install zsh -y
sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended
mkdir -p ~/starship
wget https://starship.rs/install.sh -O ~/starship/install.sh
bash ~/starship/install.sh -y
View kedro-debug-hook.py
""" Kedro Debug Hook module """
from kedro.framework.hooks import hook_impl
class debug_hook:
""" Kedro Debug Hook
Opens a debugger at any hook-able point of your kedro projects lifecycle.
debug_hook is applied by adding it to the pipeline and setting the desired
debug points to true.
View kedro-class-hook-template.py
from kedro.framework.hooks import hook_impl
class <hook_name>:
def __init__(self):
pass
@hook_impl
def before_pipeline_run(self, run_params, pipeline, catalog):
pass
View kedro-preflight.py
# full article https://waylonwalker.com/blog/creating-the-kedro-preflight-hook/
from kedro.hooks import hook_impl
from kedro.io.core import DataSetNotFoundError
from colorama import Fore
import textwrap
@hook_impl
def before_pipeline_run(run_params, pipeline, catalog):
View jupyter-onedark.css
/*-----------------------------------------------------------------------------
| Copyright (c) Jupyter Development Team.
| Distributed under the terms of the Modified BSD License.
|----------------------------------------------------------------------------*/
/*
The following CSS variables define the main, public API for styling JupyterLab.
These variables should be used by all plugins wherever possible. In other
words, plugins should not define custom colors, sizes, etc unless absolutely
necessary. This enables users to change the visual theme of JupyterLab
by changing these variables.