Last active
March 18, 2020 19:44
-
-
Save theomessin/ac84cf50ad32c1134b99de1096405e85 to your computer and use it in GitHub Desktop.
Argo Workflows volume ingress/egress using Google Cloud Storage
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
# This workflow template provides basic Google Cloud Storage I/O. | |
# Easily transfer data from Cloud Storage into Volumes and vice-versa. | |
apiVersion: argoproj.io/v1alpha1 | |
kind: WorkflowTemplate | |
metadata: | |
name: google-cloud-storage | |
spec: | |
templates: | |
# Use to get a folder from Google Cloud Storage in to a Volume. | |
# Both the gs://source and volume mount are input parameters. | |
- name: storage-to-volume | |
inputs: | |
parameters: | |
- name: source | |
- name: volume | |
container: | |
image: google/cloud-sdk:alpine | |
command: [gsutil, -m, rsync, -rd, "{{inputs.parameters.source}}", "/mnt/volume"] | |
volumeMounts: | |
- name: "{{inputs.parameters.volume}}" | |
mountPath: /mnt/volume | |
# Use to get a complete Volume folder in to Google Cloud Storage. | |
# Both the volume mount and gs://target are input parameters. | |
- name: volume-to-storage | |
inputs: | |
parameters: | |
- name: volume | |
- name: target | |
container: | |
image: google/cloud-sdk:alpine | |
command: [gsutil, -m, rsync, -rd, "/mnt/volume", "{{inputs.parameters.target}}"] | |
volumeMounts: | |
- name: "{{inputs.parameters.volume}}" | |
mountPath: /mnt/volume |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
apiVersion: argoproj.io/v1alpha1 | |
kind: Workflow | |
metadata: | |
generateName: google-cloud-storage-example- | |
spec: | |
entrypoint: workflow | |
volumeClaimTemplates: | |
- metadata: | |
name: example | |
spec: | |
accessModes: [ "ReadWriteOnce" ] | |
resources: | |
requests: | |
storage: 2Gi | |
templates: | |
- name: workflow | |
steps: | |
# Get some data from GCS to a volume | |
- - name: ingress | |
templateRef: | |
name: google-cloud-storage | |
template: storage-to-volume | |
arguments: | |
parameters: | |
- name: source | |
value: "gs://example/input/" | |
- name: volume | |
value: example | |
# Get some data from a volume to GCS | |
- - name: egress | |
templateRef: | |
name: google-cloud-storage | |
template: volume-to-storage | |
arguments: | |
parameters: | |
- name: volume | |
value: example | |
- name: target | |
value: "gs://example/output" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment