Last active
April 29, 2020 10:28
-
-
Save hnykda/47b25c1944c848df3067ec253c9553f1 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Workflow update successful" namespace=default phase=Failed resourceVersion=153252747 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Checking daemoned children of " namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Marking workflow completed" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Updated message -> pod termination" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Updated phase Running -> Failed" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="node &NodeStatus{ID:orchestra-qualityfails-core-q1-2020-20200428-230010,Name:orchestra-qualityfails-core-q1-2020-20200428-230010,DisplayName:orchestra-qualityfails-core-q1-2020-20200428-230010,Type:Retry,TemplateName:orchestra,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:pod termination,StartedAt:2020-04-28 23:00:10 +0000 UTC,FinishedAt:2020-04-28 23:11:30.356577766 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[orchestra-qualityfails-core-q1-2020-20200428-230010-960781036],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:,} finished: 2020-04-28 23:11:30.356577766 +0000 UTC" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="node &NodeStatus{ID:orchestra-qualityfails-core-q1-2020-20200428-230010,Name:orchestra-qualityfails-core-q1-2020-20200428-230010,DisplayName:orchestra-qualityfails-core-q1-2020-20200428-230010,Type:Retry,TemplateName:orchestra,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-04-28 23:00:10 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[orchestra-qualityfails-core-q1-2020-20200428-230010-960781036],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:,} message: pod termination" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="node &NodeStatus{ID:orchestra-qualityfails-core-q1-2020-20200428-230010,Name:orchestra-qualityfails-core-q1-2020-20200428-230010,DisplayName:orchestra-qualityfails-core-q1-2020-20200428-230010,Type:Retry,TemplateName:orchestra,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-04-28 23:00:10 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[orchestra-qualityfails-core-q1-2020-20200428-230010-960781036],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:,} phase Running -> Failed" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Node not set to be retried after status: Failed" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:34.961585860Z time="2020-04-28T23:11:30Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:11:21.790205163Z time="2020-04-28T23:11:16Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:10:57.985618080Z time="2020-04-28T23:10:52Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:10:40.904001972Z time="2020-04-28T23:10:34Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:09:51.739309821Z time="2020-04-28T23:09:46Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:04:23.279763960Z time="2020-04-28T23:04:17Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:01:33.970503588Z time="2020-04-28T23:01:32Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:01:33.970503588Z time="2020-04-28T23:01:31Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=153249342 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:01:33.970503588Z time="2020-04-28T23:01:31Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:01:01.838043825Z time="2020-04-28T23:00:56Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:55Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=153249068 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:55Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:55Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:53Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:52Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=153249008 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:52Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:52Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:51Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=153248969 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:56.728846144Z time="2020-04-28T23:00:51Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:12Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:11Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=153248597 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:11Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:10Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=153248591 workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:10Z" level=info msg="Created pod: orchestra-qualityfails-core-q1-2020-20200428-230010(0) (orchestra-qualityfails-core-q1-2020-20200428-230010-960781036)" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:10Z" level=info msg="Pod node {orchestra-qualityfails-core-q1-2020-20200428-230010-960781036 orchestra-qualityfails-core-q1-2020-20200428-230010(0) orchestra-qualityfails-core-q1-2020-20200428-230010(0) Pod orchestra nil Pending 2020-04-28 23:00:10.216567847 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] []} initialized Pending" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:10Z" level=info msg="Retry node {orchestra-qualityfails-core-q1-2020-20200428-230010 orchestra-qualityfails-core-q1-2020-20200428-230010 orchestra-qualityfails-core-q1-2020-20200428-230010 Retry orchestra nil Running 2020-04-28 23:00:10.216325047 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] []} initialized Running" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:10Z" level=info msg="Updated phase -> Running" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| 2020-04-28T23:00:13.729517191Z time="2020-04-28T23:00:10Z" level=info msg="Processing workflow" namespace=default workflow=orchestra-qualityfails-core-q1-2020-20200428-230010 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| protoPayload: { | |
| @type: "type.googleapis.com/google.cloud.audit.AuditLog" | |
| status: {0} | |
| authenticationInfo: { | |
| principalEmail: "system@google.com" | |
| } | |
| requestMetadata: { | |
| requestAttributes: {0} | |
| destinationAttributes: {0} | |
| } | |
| serviceName: "compute.googleapis.com" | |
| methodName: "compute.instances.preempted" | |
| resourceName: "projects/data-tooling-p/zones/europe-west1-c/instances/gke-data-tooling-p2-pyxecutor-jobs-hi-74f8d9ed-00mg" | |
| request: { | |
| @type: "type.googleapis.com/compute.instances.preempted" | |
| } | |
| } | |
| insertId: "-8vd172e7b8m8" | |
| resource: { | |
| type: "gce_instance" | |
| labels: { | |
| project_id: "data-tooling-p" | |
| instance_id: "3450722975685610385" | |
| zone: "europe-west1-c" | |
| } | |
| } | |
| timestamp: "2020-04-28T23:11:09.726Z" | |
| severity: "INFO" | |
| logName: "projects/data-tooling-p/logs/cloudaudit.googleapis.com%2Fsystem_event" | |
| operation: { | |
| id: "systemevent-1588115464305-5a461f21cbc95-12071b36-dca59a54" | |
| producer: "compute.instances.preempted" | |
| first: true | |
| last: true | |
| } | |
| receiveTimestamp: "2020-04-28T23:11:10.484752893Z" | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| apiVersion: argoproj.io/v1alpha1 | |
| kind: Workflow | |
| metadata: | |
| creationTimestamp: "2020-04-28T23:00:10Z" | |
| generateName: orchestra- | |
| generation: 8 | |
| labels: | |
| app: pyxecutor | |
| workflows.argoproj.io/completed: "true" | |
| workflows.argoproj.io/phase: Failed | |
| name: orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| namespace: default | |
| resourceVersion: "153252747" | |
| selfLink: /apis/argoproj.io/v1alpha1/namespaces/default/workflows/orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| uid: 02c05cd1-89a4-11ea-86b2-42010a8400c8 | |
| spec: | |
| arguments: | |
| parameters: | |
| - name: project_id | |
| value: core | |
| entrypoint: orchestra | |
| templates: | |
| - container: | |
| args: | |
| - -c | |
| - luigi ... | |
| command: | |
| - bash | |
| envFrom: | |
| - configMapRef: | |
| name: pyxecutor | |
| - secretRef: | |
| name: pyxecutor | |
| image: eu.gcr.io/{{workflow.parameters.gcp_project}}/pyxecutor:{{workflow.parameters.image_tag}} | |
| name: "" | |
| resources: | |
| limits: | |
| cpu: "18" | |
| memory: 109Gi | |
| requests: | |
| cpu: "1" | |
| memory: 80Gi | |
| inputs: {} | |
| metadata: | |
| labels: | |
| orchestra_job_id: '{{workflow.parameters.job_id}}' | |
| orchestra_project_id: '{{workflow.parameters.project_id}}' | |
| wave: '{{workflow.parameters.wave}}' | |
| name: orchestra | |
| outputs: {} | |
| retryStrategy: | |
| limit: 5 | |
| retryPolicy: OnError | |
| status: | |
| finishedAt: "2020-04-28T23:11:30Z" | |
| message: pod termination | |
| nodes: | |
| orchestra-qualityfails-core-q1-2020-20200428-230010: | |
| children: | |
| - orchestra-qualityfails-core-q1-2020-20200428-230010-960781036 | |
| displayName: orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| finishedAt: "2020-04-28T23:11:30Z" | |
| id: orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| message: pod termination | |
| name: orchestra-qualityfails-core-q1-2020-20200428-230010 | |
| phase: Failed | |
| startedAt: "2020-04-28T23:00:10Z" | |
| templateName: orchestra | |
| type: Retry | |
| orchestra-qualityfails-core-q1-2020-20200428-230010-960781036: | |
| displayName: orchestra-qualityfails-core-q1-2020-20200428-230010(0) | |
| finishedAt: "2020-04-28T23:11:30Z" | |
| id: orchestra-qualityfails-core-q1-2020-20200428-230010-960781036 | |
| message: pod termination | |
| name: orchestra-qualityfails-core-q1-2020-20200428-230010(0) | |
| phase: Failed | |
| startedAt: "2020-04-28T23:00:10Z" | |
| templateName: orchestra | |
| type: Pod | |
| phase: Failed | |
| startedAt: "2020-04-28T23:00:10Z" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment