Skip to content

Instantly share code, notes, and snippets.

@jyotishp

jyotishp/controller-logs.txt

Created June 17, 2020 11:34
Show Gist options
  • Save jyotishp/6ee2ce5e629f83f428bbb3d6c89df6f8 to your computer and use it in GitHub Desktop.
Save jyotishp/6ee2ce5e629f83f428bbb3d6c89df6f8 to your computer and use it in GitHub Desktop.
Download ZIP
argo-retry-timeouts
Raw
controller-logs.txt
time="2020-06-17T11:17:19Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:19Z" level=info msg="Updated phase -> Running" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:19Z" level=info msg="Retry node {retry-timeouts-mzcw8 retry-timeouts-mzcw8 retry-timeouts-mzcw8 Retry step-with-retries nil local/retry-timeouts-mzcw8 Running 2020-06-17 11:17:19.340907586 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Running" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:19Z" level=info msg="Pod node {retry-timeouts-mzcw8-3754190965 retry-timeouts-mzcw8(0) retry-timeouts-mzcw8(0) Pod step-with-retries nil local/retry-timeouts-mzcw8 Pending 2020-06-17 11:17:19.341432545 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Pending" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:19Z" level=info msg="Created pod: retry-timeouts-mzcw8(0) (retry-timeouts-mzcw8-3754190965)" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:19Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988917 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:20Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:20Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3754190965,Name:retry-timeouts-mzcw8(0),DisplayName:retry-timeouts-mzcw8(0),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: ContainerCreating"
time="2020-06-17T11:17:20Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988926 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:21Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:21Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988926 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:22Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:22Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3754190965,Name:retry-timeouts-mzcw8(0),DisplayName:retry-timeouts-mzcw8(0),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:ContainerCreating,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Pending -> Running"
time="2020-06-17T11:17:22Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988936 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:23Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:23Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988936 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:29Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:29Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3754190965,Name:retry-timeouts-mzcw8(0),DisplayName:retry-timeouts-mzcw8(0),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Running -> Failed"
time="2020-06-17T11:17:29Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3754190965,Name:retry-timeouts-mzcw8(0),DisplayName:retry-timeouts-mzcw8(0),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} message: Pod was active on the node longer than the specified deadline"
time="2020-06-17T11:17:29Z" level=info msg="1 child nodes of retry-timeouts-mzcw8 failed. Trying again..." namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:29Z" level=info msg="Pod node {retry-timeouts-mzcw8-1405574568 retry-timeouts-mzcw8(1) retry-timeouts-mzcw8(1) Pod step-with-retries nil local/retry-timeouts-mzcw8 Pending 2020-06-17 11:17:29.112497155 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Pending" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:29Z" level=info msg="Created pod: retry-timeouts-mzcw8(1) (retry-timeouts-mzcw8-1405574568)" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:29Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988970 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:30Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:30Z" level=info msg="Labeled pod default/retry-timeouts-mzcw8-3754190965 completed"
time="2020-06-17T11:17:30Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-1405574568,Name:retry-timeouts-mzcw8(1),DisplayName:retry-timeouts-mzcw8(1),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:29 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: ContainerCreating"
time="2020-06-17T11:17:30Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988979 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:31Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:31Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988979 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:32Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:32Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-1405574568,Name:retry-timeouts-mzcw8(1),DisplayName:retry-timeouts-mzcw8(1),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:ContainerCreating,StartedAt:2020-06-17 11:17:29 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Pending -> Running"
time="2020-06-17T11:17:32Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988988 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:33Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:33Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47988988 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:39Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:39Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-1405574568,Name:retry-timeouts-mzcw8(1),DisplayName:retry-timeouts-mzcw8(1),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:29 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Running -> Failed"
time="2020-06-17T11:17:39Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-1405574568,Name:retry-timeouts-mzcw8(1),DisplayName:retry-timeouts-mzcw8(1),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:29 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} message: Pod was active on the node longer than the specified deadline"
time="2020-06-17T11:17:39Z" level=info msg="2 child nodes of retry-timeouts-mzcw8 failed. Trying again..." namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:39Z" level=info msg="Pod node {retry-timeouts-mzcw8-3552565347 retry-timeouts-mzcw8(2) retry-timeouts-mzcw8(2) Pod step-with-retries nil local/retry-timeouts-mzcw8 Pending 2020-06-17 11:17:39.153291048 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Pending" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:39Z" level=info msg="Created pod: retry-timeouts-mzcw8(2) (retry-timeouts-mzcw8-3552565347)" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:39Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989020 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:40Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:40Z" level=info msg="Labeled pod default/retry-timeouts-mzcw8-1405574568 completed"
time="2020-06-17T11:17:40Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3552565347,Name:retry-timeouts-mzcw8(2),DisplayName:retry-timeouts-mzcw8(2),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:39 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: ContainerCreating"
time="2020-06-17T11:17:40Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989030 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:41Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:41Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989030 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:42Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:42Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3552565347,Name:retry-timeouts-mzcw8(2),DisplayName:retry-timeouts-mzcw8(2),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:ContainerCreating,StartedAt:2020-06-17 11:17:39 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Pending -> Running"
time="2020-06-17T11:17:42Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989040 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:43Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:43Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989040 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:49Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:49Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3552565347,Name:retry-timeouts-mzcw8(2),DisplayName:retry-timeouts-mzcw8(2),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:39 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Running -> Failed"
time="2020-06-17T11:17:49Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3552565347,Name:retry-timeouts-mzcw8(2),DisplayName:retry-timeouts-mzcw8(2),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:39 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} message: Pod was active on the node longer than the specified deadline"
time="2020-06-17T11:17:49Z" level=info msg="3 child nodes of retry-timeouts-mzcw8 failed. Trying again..." namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:49Z" level=info msg="Pod node {retry-timeouts-mzcw8-3082939110 retry-timeouts-mzcw8(3) retry-timeouts-mzcw8(3) Pod step-with-retries nil local/retry-timeouts-mzcw8 Pending 2020-06-17 11:17:49.1327189 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Pending" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:49Z" level=info msg="Created pod: retry-timeouts-mzcw8(3) (retry-timeouts-mzcw8-3082939110)" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:49Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989075 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:50Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:50Z" level=info msg="Labeled pod default/retry-timeouts-mzcw8-3552565347 completed"
time="2020-06-17T11:17:50Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3082939110,Name:retry-timeouts-mzcw8(3),DisplayName:retry-timeouts-mzcw8(3),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:49 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: ContainerCreating"
time="2020-06-17T11:17:50Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989084 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:51Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:51Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989084 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:52Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:52Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3082939110,Name:retry-timeouts-mzcw8(3),DisplayName:retry-timeouts-mzcw8(3),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:ContainerCreating,StartedAt:2020-06-17 11:17:49 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Pending -> Running"
time="2020-06-17T11:17:52Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989093 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:53Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:53Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989093 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:59Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:59Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3082939110,Name:retry-timeouts-mzcw8(3),DisplayName:retry-timeouts-mzcw8(3),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:49 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Running -> Failed"
time="2020-06-17T11:17:59Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3082939110,Name:retry-timeouts-mzcw8(3),DisplayName:retry-timeouts-mzcw8(3),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:49 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} message: Pod was active on the node longer than the specified deadline"
time="2020-06-17T11:17:59Z" level=info msg="4 child nodes of retry-timeouts-mzcw8 failed. Trying again..." namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:59Z" level=info msg="Pod node {retry-timeouts-mzcw8-3083777753 retry-timeouts-mzcw8(4) retry-timeouts-mzcw8(4) Pod step-with-retries nil local/retry-timeouts-mzcw8 Pending 2020-06-17 11:17:59.120707997 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Pending" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:59Z" level=info msg="Created pod: retry-timeouts-mzcw8(4) (retry-timeouts-mzcw8-3083777753)" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:17:59Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989125 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:00Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:00Z" level=info msg="Labeled pod default/retry-timeouts-mzcw8-3082939110 completed"
time="2020-06-17T11:18:00Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3083777753,Name:retry-timeouts-mzcw8(4),DisplayName:retry-timeouts-mzcw8(4),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:59 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: ContainerCreating"
time="2020-06-17T11:18:00Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989135 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:01Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:01Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3083777753,Name:retry-timeouts-mzcw8(4),DisplayName:retry-timeouts-mzcw8(4),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:ContainerCreating,StartedAt:2020-06-17 11:17:59 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Pending -> Running"
time="2020-06-17T11:18:01Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989140 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:02Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:02Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989140 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:09Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:09Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3083777753,Name:retry-timeouts-mzcw8(4),DisplayName:retry-timeouts-mzcw8(4),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:59 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Running -> Failed"
time="2020-06-17T11:18:09Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3083777753,Name:retry-timeouts-mzcw8(4),DisplayName:retry-timeouts-mzcw8(4),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:59 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} message: Pod was active on the node longer than the specified deadline"
time="2020-06-17T11:18:09Z" level=info msg="5 child nodes of retry-timeouts-mzcw8 failed. Trying again..." namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:09Z" level=info msg="Pod node {retry-timeouts-mzcw8-3687919132 retry-timeouts-mzcw8(5) retry-timeouts-mzcw8(5) Pod step-with-retries nil local/retry-timeouts-mzcw8 Pending 2020-06-17 11:18:09.122183595 +0000 UTC 0001-01-01 00:00:00 +0000 UTC <nil> nil nil [] [] } initialized Pending" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:09Z" level=info msg="Created pod: retry-timeouts-mzcw8(5) (retry-timeouts-mzcw8-3687919132)" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:09Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989175 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:10Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:10Z" level=info msg="Labeled pod default/retry-timeouts-mzcw8-3083777753 completed"
time="2020-06-17T11:18:10Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3687919132,Name:retry-timeouts-mzcw8(5),DisplayName:retry-timeouts-mzcw8(5),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:,StartedAt:2020-06-17 11:18:09 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: ContainerCreating"
time="2020-06-17T11:18:10Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989186 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:11Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:11Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989186 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:12Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:12Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3687919132,Name:retry-timeouts-mzcw8(5),DisplayName:retry-timeouts-mzcw8(5),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Pending,BoundaryID:,Message:ContainerCreating,StartedAt:2020-06-17 11:18:09 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Pending -> Running"
time="2020-06-17T11:18:12Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989195 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:13Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:13Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion=47989195 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Processing workflow" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3687919132,Name:retry-timeouts-mzcw8(5),DisplayName:retry-timeouts-mzcw8(5),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:18:09 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} status Running -> Failed"
time="2020-06-17T11:18:19Z" level=info msg="Updating node &NodeStatus{ID:retry-timeouts-mzcw8-3687919132,Name:retry-timeouts-mzcw8(5),DisplayName:retry-timeouts-mzcw8(5),Type:Pod,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:18:09 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:internal-node,} message: Pod was active on the node longer than the specified deadline"
time="2020-06-17T11:18:19Z" level=info msg="No more retries left. Failing..." namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="node &NodeStatus{ID:retry-timeouts-mzcw8,Name:retry-timeouts-mzcw8,DisplayName:retry-timeouts-mzcw8,Type:Retry,TemplateName:step-with-retries,TemplateRef:nil,Phase:Running,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[retry-timeouts-mzcw8-3754190965 retry-timeouts-mzcw8-1405574568 retry-timeouts-mzcw8-3552565347 retry-timeouts-mzcw8-3082939110 retry-timeouts-mzcw8-3083777753 retry-timeouts-mzcw8-3687919132],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} phase Running -> Failed" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="node &NodeStatus{ID:retry-timeouts-mzcw8,Name:retry-timeouts-mzcw8,DisplayName:retry-timeouts-mzcw8,Type:Retry,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:0001-01-01 00:00:00 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[retry-timeouts-mzcw8-3754190965 retry-timeouts-mzcw8-1405574568 retry-timeouts-mzcw8-3552565347 retry-timeouts-mzcw8-3082939110 retry-timeouts-mzcw8-3083777753 retry-timeouts-mzcw8-3687919132],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} message: No more retries left" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="node &NodeStatus{ID:retry-timeouts-mzcw8,Name:retry-timeouts-mzcw8,DisplayName:retry-timeouts-mzcw8,Type:Retry,TemplateName:step-with-retries,TemplateRef:nil,Phase:Failed,BoundaryID:,Message:No more retries left,StartedAt:2020-06-17 11:17:19 +0000 UTC,FinishedAt:2020-06-17 11:18:19.142078309 +0000 UTC,PodIP:,Daemoned:nil,Inputs:nil,Outputs:nil,Children:[retry-timeouts-mzcw8-3754190965 retry-timeouts-mzcw8-1405574568 retry-timeouts-mzcw8-3552565347 retry-timeouts-mzcw8-3082939110 retry-timeouts-mzcw8-3083777753 retry-timeouts-mzcw8-3687919132],OutboundNodes:[],StoredTemplateID:,WorkflowTemplateName:,TemplateScope:local/retry-timeouts-mzcw8,ResourcesDuration:ResourcesDuration{},HostNodeName:,} finished: 2020-06-17 11:18:19.142078309 +0000 UTC" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Updated phase Running -> Failed" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Updated message -> No more retries left" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Marking workflow completed" namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Checking daemoned children of " namespace=default workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:19Z" level=info msg="Workflow update successful" namespace=default phase=Failed resourceVersion=47989226 workflow=retry-timeouts-mzcw8
time="2020-06-17T11:18:20Z" level=info msg="Labeled pod default/retry-timeouts-mzcw8-3687919132 completed"
Raw
workflow-output.txt
Name: retry-timeouts-mzcw8
Namespace: default
ServiceAccount: default
Status: Failed
Message: No more retries left
Conditions:
Completed True
Created: Wed Jun 17 16:47:19 +0530 (10 minutes ago)
Started: Wed Jun 17 16:47:19 +0530 (10 minutes ago)
Finished: Wed Jun 17 16:48:19 +0530 (9 minutes ago)
Duration: 1 minute 0 seconds
STEP TEMPLATE PODNAME DURATION MESSAGE
✖ retry-timeouts-mzcw8 step-with-retries No more retries left
├-✖ retry-timeouts-mzcw8(0) step-with-retries retry-timeouts-mzcw8-3754190965 10s Pod was active on the node longer than the specified deadline
├-✖ retry-timeouts-mzcw8(1) step-with-retries retry-timeouts-mzcw8-1405574568 10s Pod was active on the node longer than the specified deadline
├-✖ retry-timeouts-mzcw8(2) step-with-retries retry-timeouts-mzcw8-3552565347 10s Pod was active on the node longer than the specified deadline
├-✖ retry-timeouts-mzcw8(3) step-with-retries retry-timeouts-mzcw8-3082939110 10s Pod was active on the node longer than the specified deadline
├-✖ retry-timeouts-mzcw8(4) step-with-retries retry-timeouts-mzcw8-3083777753 10s Pod was active on the node longer than the specified deadline
└-✖ retry-timeouts-mzcw8(5) step-with-retries retry-timeouts-mzcw8-3687919132 10s Pod was active on the node longer than the specified deadline
Raw
workflow-spec.yaml
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: retry-timeouts-
spec:
entrypoint: step-with-retries
templates:
- name: step-with-retries
activeDeadlineSeconds: 10
retryStrategy:
limit: 5
retryPolicy: "Always"
container:
image: alpine
command: [sh, -c]
args:
- |
sleep 30
Raw
workflow-status.yaml
metadata:
creationTimestamp: "2020-06-17T11:17:19Z"
generateName: retry-timeouts-
generation: 20
labels:
workflows.argoproj.io/completed: "true"
workflows.argoproj.io/phase: Failed
name: retry-timeouts-mzcw8
namespace: default
resourceVersion: "47989226"
selfLink: /apis/argoproj.io/v1alpha1/namespaces/default/workflows/retry-timeouts-mzcw8
uid: 1b9b5148-b08c-11ea-af3e-42010aa600f9
spec:
arguments: {}
entrypoint: step-with-retries
templates:
- activeDeadlineSeconds: 10
arguments: {}
container:
args:
- |
sleep 30
command:
- sh
- -c
image: alpine
name: ""
resources: {}
inputs: {}
metadata: {}
name: step-with-retries
outputs: {}
retryStrategy:
limit: 5
retryPolicy: Always
status:
conditions:
- status: "True"
type: Completed
finishedAt: "2020-06-17T11:18:19Z"
message: No more retries left
nodes:
retry-timeouts-mzcw8:
children:
- retry-timeouts-mzcw8-3754190965
- retry-timeouts-mzcw8-1405574568
- retry-timeouts-mzcw8-3552565347
- retry-timeouts-mzcw8-3082939110
- retry-timeouts-mzcw8-3083777753
- retry-timeouts-mzcw8-3687919132
displayName: retry-timeouts-mzcw8
finishedAt: "2020-06-17T11:18:19Z"
id: retry-timeouts-mzcw8
message: No more retries left
name: retry-timeouts-mzcw8
phase: Failed
startedAt: "2020-06-17T11:17:19Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Retry
retry-timeouts-mzcw8-1405574568:
displayName: retry-timeouts-mzcw8(1)
finishedAt: "2020-06-17T11:17:39Z"
hostNodeName: internal-node
id: retry-timeouts-mzcw8-1405574568
message: Pod was active on the node longer than the specified deadline
name: retry-timeouts-mzcw8(1)
phase: Failed
startedAt: "2020-06-17T11:17:29Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Pod
retry-timeouts-mzcw8-3082939110:
displayName: retry-timeouts-mzcw8(3)
finishedAt: "2020-06-17T11:17:59Z"
hostNodeName: internal-node
id: retry-timeouts-mzcw8-3082939110
message: Pod was active on the node longer than the specified deadline
name: retry-timeouts-mzcw8(3)
phase: Failed
startedAt: "2020-06-17T11:17:49Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Pod
retry-timeouts-mzcw8-3083777753:
displayName: retry-timeouts-mzcw8(4)
finishedAt: "2020-06-17T11:18:09Z"
hostNodeName: internal-node
id: retry-timeouts-mzcw8-3083777753
message: Pod was active on the node longer than the specified deadline
name: retry-timeouts-mzcw8(4)
phase: Failed
startedAt: "2020-06-17T11:17:59Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Pod
retry-timeouts-mzcw8-3552565347:
displayName: retry-timeouts-mzcw8(2)
finishedAt: "2020-06-17T11:17:49Z"
hostNodeName: internal-node
id: retry-timeouts-mzcw8-3552565347
message: Pod was active on the node longer than the specified deadline
name: retry-timeouts-mzcw8(2)
phase: Failed
startedAt: "2020-06-17T11:17:39Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Pod
retry-timeouts-mzcw8-3687919132:
displayName: retry-timeouts-mzcw8(5)
finishedAt: "2020-06-17T11:18:19Z"
hostNodeName: internal-node
id: retry-timeouts-mzcw8-3687919132
message: Pod was active on the node longer than the specified deadline
name: retry-timeouts-mzcw8(5)
phase: Failed
startedAt: "2020-06-17T11:18:09Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Pod
retry-timeouts-mzcw8-3754190965:
displayName: retry-timeouts-mzcw8(0)
finishedAt: "2020-06-17T11:17:29Z"
hostNodeName: internal-node
id: retry-timeouts-mzcw8-3754190965
message: Pod was active on the node longer than the specified deadline
name: retry-timeouts-mzcw8(0)
phase: Failed
startedAt: "2020-06-17T11:17:19Z"
templateName: step-with-retries
templateScope: local/retry-timeouts-mzcw8
type: Pod
phase: Failed
startedAt: "2020-06-17T11:17:19Z"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment