Skip to content

Instantly share code, notes, and snippets.

@Wats0ns
Created January 24, 2020 10:54
Show Gist options
  • Save Wats0ns/a5d31330af705a20a4b3e3f76952bfca to your computer and use it in GitHub Desktop.
Save Wats0ns/a5d31330af705a20a4b3e3f76952bfca to your computer and use it in GitHub Desktop.
kubeflow.yaml
"apiVersion": |-
argoproj.io/v1alpha1
"kind": |-
Workflow
"metadata":
"annotations":
"pipelines.kubeflow.org/pipeline_spec": |-
{"description": "A pipeline with two sequential steps.", "inputs": [{"default": "gs://kubeflow-ryanbouamara/kc_house_data.csv", "name": "url", "optional": true}, {"default": "/mnt/trained_model", "name": "trained_path", "optional": true}, {"default": "/mnt/dataset.csv", "name": "data_path", "optional": true}], "name": "Sequential pipeline"}
"generateName": |-
sequential-pipeline-
"spec":
"arguments":
"parameters":
- "name": |-
url
"value": |-
gs://kubeflow-ryanbouamara/kc_house_data.csv
- "name": |-
trained_path
"value": |-
/mnt/trained_model
- "name": |-
data_path
"value": |-
/mnt/dataset.csv
"entrypoint": |-
sequential-pipeline
"serviceAccountName": |-
pipeline-runner
"templates":
- "name": |-
creation-of-volume
"outputs":
"parameters":
- "name": |-
creation-of-volume-manifest
"valueFrom":
"jsonPath": |-
{}
- "name": |-
creation-of-volume-name
"valueFrom":
"jsonPath": |-
{.metadata.name}
- "name": |-
creation-of-volume-size
"valueFrom":
"jsonPath": |-
{.status.capacity.storage}
"resource":
"action": |-
create
"manifest": |
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: '{{workflow.name}}-dataset-pvc'
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
- "container":
"args":
- |-
gsutil cat $0 | tee $1
- |-
{{inputs.parameters.url}}
- |-
{{inputs.parameters.data_path}}
"command":
- |-
sh
- |-
-c
"image": |-
google/cloud-sdk:272.0.0
"resources":
"limits":
"cpu": |-
1
"memory": |-
2G
"requests":
"cpu": |-
0.5
"memory": |-
1G
"volumeMounts":
- "mountPath": |-
/mnt
"name": |-
creation-of-volume
"inputs":
"parameters":
- "name": |-
creation-of-volume-name
- "name": |-
data_path
- "name": |-
url
"name": |-
download-dataset
"volumes":
- "name": |-
creation-of-volume
"persistentVolumeClaim":
"claimName": |-
{{inputs.parameters.creation-of-volume-name}}
- "container":
"args":
- |-
python3 preprocess.py
"command":
- |-
sh
- |-
-c
"env":
- "name": |-
TRAINED_MODEL_PATH
"value": |-
{{inputs.parameters.trained_path}}
- "name": |-
DATA_PATH
"value": |-
{{inputs.parameters.data_path}}
"image": |-
anyr/workshop:python2
"resources":
"limits":
"cpu": |-
1
"memory": |-
4G
"requests":
"cpu": |-
1
"memory": |-
2G
"volumeMounts":
- "mountPath": |-
/mnt
"name": |-
creation-of-volume
"inputs":
"parameters":
- "name": |-
creation-of-volume-name
- "name": |-
data_path
- "name": |-
trained_path
"name": |-
preprocess-step
"volumes":
- "name": |-
creation-of-volume
"persistentVolumeClaim":
"claimName": |-
{{inputs.parameters.creation-of-volume-name}}
- "dag":
"tasks":
- "name": |-
creation-of-volume
"template": |-
creation-of-volume
- "arguments":
"parameters":
- "name": |-
creation-of-volume-name
"value": |-
{{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
- "name": |-
data_path
"value": |-
{{inputs.parameters.data_path}}
- "name": |-
url
"value": |-
{{inputs.parameters.url}}
"dependencies":
- |-
creation-of-volume
"name": |-
download-dataset
"template": |-
download-dataset
- "arguments":
"parameters":
- "name": |-
creation-of-volume-name
"value": |-
{{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
- "name": |-
data_path
"value": |-
{{inputs.parameters.data_path}}
- "name": |-
trained_path
"value": |-
{{inputs.parameters.trained_path}}
"dependencies":
- |-
creation-of-volume
- |-
download-dataset
"name": |-
preprocess-step
"template": |-
preprocess-step
- "arguments":
"parameters":
- "name": |-
creation-of-volume-name
"value": |-
{{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
- "name": |-
data_path
"value": |-
{{inputs.parameters.data_path}}
- "name": |-
trained_path
"value": |-
{{inputs.parameters.trained_path}}
"dependencies":
- |-
creation-of-volume
- |-
preprocess-step
"name": |-
train-step
"template": |-
train-step
- "arguments":
"parameters":
- "name": |-
creation-of-volume-name
"value": |-
{{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
- "name": |-
data_path
"value": |-
{{inputs.parameters.data_path}}
- "name": |-
trained_path
"value": |-
{{inputs.parameters.trained_path}}
"dependencies":
- |-
creation-of-volume
- |-
train-step
"name": |-
val-step
"template": |-
val-step
"inputs":
"parameters":
- "name": |-
data_path
- "name": |-
trained_path
- "name": |-
url
"name": |-
sequential-pipeline
- "container":
"args":
- |-
python3 train.py
"command":
- |-
sh
- |-
-c
"env":
- "name": |-
TRAINED_MODEL_PATH
"value": |-
{{inputs.parameters.trained_path}}
- "name": |-
DATA_PATH
"value": |-
{{inputs.parameters.data_path}}
"image": |-
anyr/workshop:python2
"resources":
"limits":
"cpu": |-
10
"memory": |-
4G
"requests":
"cpu": |-
5
"memory": |-
2G
"volumeMounts":
- "mountPath": |-
/mnt
"name": |-
creation-of-volume
"inputs":
"parameters":
- "name": |-
creation-of-volume-name
- "name": |-
data_path
- "name": |-
trained_path
"name": |-
train-step
"volumes":
- "name": |-
creation-of-volume
"persistentVolumeClaim":
"claimName": |-
{{inputs.parameters.creation-of-volume-name}}
- "container":
"args":
- |-
python3 val.py; ls /
"command":
- |-
sh
- |-
-c
"env":
- "name": |-
TRAINED_MODEL_PATH
"value": |-
{{inputs.parameters.trained_path}}
- "name": |-
DATA_PATH
"value": |-
{{inputs.parameters.data_path}}
"image": |-
anyr/workshop:python2
"resources":
"limits":
"cpu": |-
1
"memory": |-
4G
"requests":
"cpu": |-
1
"memory": |-
2G
"volumeMounts":
- "mountPath": |-
/mnt
"name": |-
creation-of-volume
"inputs":
"parameters":
- "name": |-
creation-of-volume-name
- "name": |-
data_path
- "name": |-
trained_path
"name": |-
val-step
"volumes":
- "name": |-
creation-of-volume
"persistentVolumeClaim":
"claimName": |-
{{inputs.parameters.creation-of-volume-name}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment