Google Workflow with Retry

google_workflow_with_retry.jaml

main:
    params: [event]
    steps:
    - init:
        assign:
          - parent: projects/[your-project]/locations/europe-west4/repositories/[your-repo]
          - extract_tableId_step1: '${json.decode(base64.decode(event.data.message.data))}'
          - extract_tableId_step2: '${extract_tableId_step1.protoPayload.metadata.tableCreation.table.tableName}'
          - extract_tableId_step3: '${text.split(extract_tableId_step2, "/")}'
          - extract_tableId_step4: '${extract_tableId_step3[len(extract_tableId_step3) -1]}'
    - log:
        call: sys.log
        args:
            text: ${extract_tableId_step4}
    - createCompilationResult:
        try:
            call: http.post
            args:
                url: ${"https://dataform.googleapis.com/v1beta1/" + parent + "/compilationResults"}
                auth:
                    type: OAuth2
                body:
                    gitCommitish: main
                    code_compilation_config:
                        vars: 
                            GA4_TABLE: ${extract_tableId_step4}
            result: compilationResult
        retry: 
            predicate: ${custom_retry_policy}
            max_retries: 10
            backoff:
                initial_delay: 1
                max_delay: 90
                multiplier: 2
    - createWorkflowInvocation:
        try:
            call: http.post
            args:
                url: ${"https://dataform.googleapis.com/v1beta1/" + parent + "/workflowInvocations"}
                auth:
                    type: OAuth2
                body:
                    compilationResult: ${compilationResult.body.name}
                    invocationConfig: 
                        fullyRefreshIncrementalTablesEnabled: false
                        includedTags: ["ga4"]
                        includedTargets: []
                        transitiveDependenciesIncluded: true
                        transitiveDependentsIncluded: false
            result: workflowInvocation
        retry: 
            predicate: ${custom_retry_policy}
            max_retries: 10
            backoff:
                initial_delay: 1
                max_delay: 90
                multiplier: 2
    - complete:
        return: ${workflowInvocation.body.name + " complete"}

custom_retry_policy:
    params: [e]
    steps:
    - what_to_repeat:
        switch:
          - condition: ${e.code == 400}
            return: true
          - condition: ${e.code == 429}
            return: true
          - condition: ${e.code == 502}
            return: true
          - condition: ${e.code == 503}
            return: true
          - condition: ${e.code == 504}
            return: true
    - otherwise:
        return: false