Skip to content

Instantly share code, notes, and snippets.

View gregsheremeta's full-sized avatar

Greg Sheremeta gregsheremeta

17 followers · 0 following
View GitHub Profile
@gregsheremeta
gregsheremeta / iris-training-pipeline.py
Created March 28, 2024 19:04
iris-training-pipeline.py
from typing import List
from kfp import client
from kfp import compiler
from kfp import dsl
from kfp.dsl import Dataset
from kfp.dsl import Input
from kfp.dsl import Model
from kfp.dsl import Output
@gregsheremeta
gregsheremeta / iris-training-pipeline.yaml
Created March 28, 2024 19:03
iris-training-pipeline.yaml
# PIPELINE DEFINITION
# Name: iris-training-pipeline
# Inputs:
# min_max_scaler: bool
# neighbors: int
# standard_scaler: bool
components:
comp-create-dataset:
executorLabel: exec-create-dataset
outputDefinitions:
@gregsheremeta
gregsheremeta / automl-tabular.yaml
Created March 28, 2024 18:59
automl-tabular.yaml
# PIPELINE DEFINITION
# Name: automl-tabular
# Description: The AutoML Tabular pipeline v1.
# Inputs:
# additional_experiments: dict
# cv_trainer_worker_pool_specs_override: list
# data_source_bigquery_table_path: str [Default: '']
# data_source_csv_filenames: str [Default: '']
# dataflow_service_account: str [Default: '']
# dataflow_subnetwork: str [Default: '']
@gregsheremeta
gregsheremeta / nested-conditions.yaml
Created March 28, 2024 18:54
nested-conditions.yaml
# PIPELINE DEFINITION
# Name: nested-conditions-pipeline
components:
comp-condition-1:
dag:
tasks:
condition-2:
componentRef:
name: comp-condition-2
dependentTasks:
@gregsheremeta
gregsheremeta / a6.py
Last active July 17, 2023 21:11
a6 - same as a1, but with empty string in the OutputPath
"""Test pipeline to exercise various data flow mechanisms."""
import kfp
"""Producer"""
def send_file(
outgoingfile: kfp.components.OutputPath(),
):
import urllib.request
@gregsheremeta
gregsheremeta / a3.py
Last active July 17, 2023 21:03
a3 - same as a1, but no string in OutputPath
"""Test pipeline to exercise various data flow mechanisms."""
import kfp
"""Producer"""
def send_file(
outgoingfile: kfp.components.OutputPath(),
):
import urllib.request
@gregsheremeta
gregsheremeta / a5.py
Last active July 17, 2023 21:11
a5 - same as a2, but with empty string in OutputPath
"""Test pipeline to exercise various data flow mechanisms."""
import kfp
"""Producer"""
def send_file(
outgoingfile: kfp.components.OutputPath(),
):
import urllib.request
@gregsheremeta
gregsheremeta / a4.py
Last active July 17, 2023 21:11
a4 - same as a2, but with no string in the OutputPath
"""Test pipeline to exercise various data flow mechanisms."""
import kfp
"""Producer"""
def send_file(
outgoingfile: kfp.components.OutputPath(),
):
import urllib.request
@gregsheremeta
gregsheremeta / a2.py
Last active July 17, 2023 21:12
a2 - without add_pod_annotation - 'Termination message is above max allowed size 4096' error, but 20MB result exists in minio
"""Test pipeline to exercise various data flow mechanisms."""
import kfp
"""Producer"""
def send_file(
outgoingfile: kfp.components.OutputPath(),
):
import urllib.request
@gregsheremeta
gregsheremeta / a1.py
Last active July 17, 2023 21:12
a1 - with add_pod_annotation - success, but 0 byte result
"""Test pipeline to exercise various data flow mechanisms."""
import kfp
"""Producer"""
def send_file(
outgoingfile: kfp.components.OutputPath(),
):
import urllib.request