Skip to content

Instantly share code, notes, and snippets.

View WaylonWalker's full-sized avatar
💌
waylonwalker.com/newsletter

Waylon Walker WaylonWalker

💌
waylonwalker.com/newsletter
View GitHub Profile
# Never enforce `E501` (line length violations).
ignore = ["E501"]
target-version = "py37"
select = [
"F", # Pyflakes
"E", # Error
"W", # Warning
"C90", # mccabe
@WaylonWalker
WaylonWalker / dynamic_pydantic_models.py
Last active March 15, 2023 17:38
The idea is to make pluggable pydantic models where other libs can have config within a libraries config, and be type checked.
from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel, create_model, validator
import toml
class User(BaseModel):
id: int
name = "John Doe"
@WaylonWalker
WaylonWalker / first_nodes.py
Created September 12, 2021 18:56
Write Your First Pipeline Node
"""
first_nodes.py
used in video https://youtu.be/-gEwU-MrPuA
Full Post 👉 https://waylonwalker.com/kedro-your-first-nodes/
"""
from kedro.pipeline import node
from IPython.terminal.prompts import Prompts, Token
from pathlib import Path
import os
from platform import python_version
import subprocess
def get_branch():
try:
return (
subprocess.check_output(
"""Standard Config.
A module to load tooling config from a users project space.
One day this may become a package, for now its a gist.
Inspired from frustrations that some tools have a tool.ini, .tool.ini,
setup.cfg, or pyproject.toml. Some allow for global configs, some don't. Some
properly follow the users home directory, others end up in a weird temp
directory. Windows home directory is only more confusing. Some will even
respect the users `$XDG_HOME` directory.
import re
import pyperclip
def copy():
with open('C:/Debian/rootfs/home/<user>/.easyclip') as f:
easyclip = f.readlines(1)
first = re.sub(r"^.*text': '", "", easyclip[0])
last = re.sub(r"'}\n$", "", first)
#!/bin/bash
# curl -s -L https://gist.githubusercontent.com/WaylonWalker/0532e9e9f41432d9a375490235286cd6/raw/4bc499518889aa9ad8ee0e410bc44f96a860f1e0/KedroBasics_startup.sh | bash
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get upgrade -yq
apt install zsh -y
sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended
mkdir -p ~/starship
wget https://starship.rs/install.sh -O ~/starship/install.sh
bash ~/starship/install.sh -y
""" Kedro Debug Hook module """
from kedro.framework.hooks import hook_impl
class debug_hook:
""" Kedro Debug Hook
Opens a debugger at any hook-able point of your kedro projects lifecycle.
debug_hook is applied by adding it to the pipeline and setting the desired
debug points to true.
from kedro.framework.hooks import hook_impl
class <hook_name>:
def __init__(self):
pass
@hook_impl
def before_pipeline_run(self, run_params, pipeline, catalog):
pass
# full article https://waylonwalker.com/blog/creating-the-kedro-preflight-hook/
from kedro.hooks import hook_impl
from kedro.io.core import DataSetNotFoundError
from colorama import Fore
import textwrap
@hook_impl
def before_pipeline_run(run_params, pipeline, catalog):