Skip to content

Instantly share code, notes, and snippets.

@cezarsa
Created August 29, 2017 17:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save cezarsa/f5ab23ad975cc47b488c15e4969c5936 to your computer and use it in GitHub Desktop.
Save cezarsa/f5ab23ad975cc47b488c15e4969c5936 to your computer and use it in GitHub Desktop.
diff --git a/provision/kubernetes/deploy.go b/provision/kubernetes/deploy.go
index 587eac2e0..f729c2682 100644
--- a/provision/kubernetes/deploy.go
+++ b/provision/kubernetes/deploy.go
@@ -455,8 +455,12 @@ func monitorDeployment(client *clusterClient, dep *v1beta1.Deployment, a provisi
fmt.Fprintf(w, " ---> %d of %d new units created\n", dep.Status.UpdatedReplicas, specReplicas)
}
if healthcheckTimeout == nil && dep.Status.UpdatedReplicas == specReplicas {
- healthcheckTimeout = time.After(maxWaitTimeDuration)
- fmt.Fprintf(w, " ---> waiting healthcheck on %d created units\n", specReplicas)
+ var allInit bool
+ allInit, err = allPodsInitialized(client, a, processName)
+ if allInit && err == nil {
+ healthcheckTimeout = time.After(maxWaitTimeDuration)
+ fmt.Fprintf(w, " ---> waiting healthcheck on %d created units\n", specReplicas)
+ }
}
readyUnits := dep.Status.UpdatedReplicas - dep.Status.UnavailableReplicas
if oldReadyUnits != readyUnits && readyUnits >= 0 {
diff --git a/provision/kubernetes/deploy_test.go b/provision/kubernetes/deploy_test.go
index 8ddf6fe84..1f78205ba 100644
--- a/provision/kubernetes/deploy_test.go
+++ b/provision/kubernetes/deploy_test.go
@@ -622,11 +622,11 @@ func (s *S) TestServiceManagerDeployServiceRollback(c *check.C) {
err = servicecommon.RunServicePipeline(&m, a, "myimg", servicecommon.ProcessSpec{
"p1": servicecommon.ProcessState{Start: true},
})
- c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units$")
+ c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units: Pod myapp-p1-pod-1-1: invalid pod phase \"Running\"$")
c.Assert(rollbackObj, check.DeepEquals, &v1beta1.DeploymentRollback{
Name: "myapp-p1",
})
- c.Assert(buf.String(), check.Matches, `(?s).*---- Updating units \[p1\] ----.*ROLLING BACK AFTER FAILURE.*---> timeout after .* waiting for units <---\s*$`)
+ c.Assert(buf.String(), check.Matches, `(?s).*---- Updating units \[p1\] ----.*ROLLING BACK AFTER FAILURE.*---> timeout after .* waiting for units: Pod myapp-p1-pod-1-1: invalid pod phase \"Running\" <---\s*$`)
cleanupDeployment(s.client.clusterClient, a, "p1")
_, err = s.client.Core().Events(s.client.Namespace()).Create(&apiv1.Event{
ObjectMeta: metav1.ObjectMeta{
@@ -640,7 +640,7 @@ func (s *S) TestServiceManagerDeployServiceRollback(c *check.C) {
err = servicecommon.RunServicePipeline(&m, a, "myimg", servicecommon.ProcessSpec{
"p1": servicecommon.ProcessState{Start: true},
})
- c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units: Pod myapp-p1-pod-2-1: Unhealthy - my evt message$")
+ c.Assert(err, check.ErrorMatches, "^timeout after .+ waiting for units: Pod myapp-p1-pod-2-1: invalid pod phase \"Running\" - last event: my evt message$")
}
func (s *S) TestServiceManagerRemoveService(c *check.C) {
diff --git a/provision/kubernetes/helpers.go b/provision/kubernetes/helpers.go
index 784cf01ee..f6dc18132 100644
--- a/provision/kubernetes/helpers.go
+++ b/provision/kubernetes/helpers.go
@@ -94,50 +94,67 @@ func waitFor(timeout time.Duration, fn func() (bool, error), onTimeout func() er
}
}
+func podsForAppProcess(client *clusterClient, a provision.App, process string) (*apiv1.PodList, error) {
+ labelOpts := provision.ServiceLabelsOpts{
+ App: a,
+ Process: process,
+ ServiceLabelExtendedOpts: provision.ServiceLabelExtendedOpts{
+ Prefix: tsuruLabelPrefix,
+ Provisioner: provisionerName,
+ },
+ }
+ l, err := provision.ServiceLabels(labelOpts)
+ if err != nil {
+ return nil, errors.WithStack(err)
+ }
+ var selector map[string]string
+ if process == "" {
+ selector = l.ToAppSelector()
+ } else {
+ selector = l.ToSelector()
+ }
+ return client.Core().Pods(client.Namespace()).List(metav1.ListOptions{
+ LabelSelector: labels.SelectorFromSet(labels.Set(selector)).String(),
+ })
+}
+
+func allPodsInitialized(client *clusterClient, a provision.App, process string) (bool, error) {
+ pods, err := podsForAppProcess(client, a, process)
+ if err != nil {
+ return false, errors.WithStack(err)
+ }
+ for _, pod := range pods.Items {
+ for _, cond := range pod.Status.Conditions {
+ if cond.Type != apiv1.PodInitialized {
+ continue
+ }
+ if cond.Status != apiv1.ConditionTrue {
+ return false, nil
+ }
+ }
+ }
+ return true, nil
+}
+
func notReadyPodEvents(client *clusterClient, a provision.App, process string) ([]string, error) {
- l, err := provision.ServiceLabels(provision.ServiceLabelsOpts{
- App: a,
- Process: process,
- ServiceLabelExtendedOpts: provision.ServiceLabelExtendedOpts{
- Prefix: tsuruLabelPrefix,
- Provisioner: provisionerName,
- },
- })
+ pods, err := podsForAppProcess(client, a, process)
if err != nil {
return nil, errors.WithStack(err)
}
- pods, err := client.Core().Pods(client.Namespace()).List(metav1.ListOptions{
- LabelSelector: labels.SelectorFromSet(labels.Set(l.ToSelector())).String(),
- })
- if err != nil {
- return nil, errors.WithStack(err)
- }
- var podsForEvts []string
+ var podsForEvts []*apiv1.Pod
podsLoop:
- for _, pod := range pods.Items {
+ for i, pod := range pods.Items {
for _, cond := range pod.Status.Conditions {
if cond.Type == apiv1.PodReady && cond.Status != apiv1.ConditionTrue {
- podsForEvts = append(podsForEvts, pod.Name)
+ podsForEvts = append(podsForEvts, &pods.Items[i])
continue podsLoop
}
}
}
var messages []string
- for _, podName := range podsForEvts {
- eventsInterface := client.Core().Events(client.Namespace())
- ns := client.Namespace()
- selector := eventsInterface.GetFieldSelector(&podName, &ns, nil, nil)
- options := metav1.ListOptions{FieldSelector: selector.String()}
- var events *apiv1.EventList
- events, err = eventsInterface.List(options)
- if err != nil {
- return nil, errors.WithStack(err)
- }
- if len(events.Items) == 0 {
- continue
- }
- lastEvt := events.Items[len(events.Items)-1]
- messages = append(messages, fmt.Sprintf("Pod %s: %s - %s", podName, lastEvt.Reason, lastEvt.Message))
+ for _, pod := range podsForEvts {
+ err = newInvalidPodPhaseError(client, pod)
+ messages = append(messages, fmt.Sprintf("Pod %s: %v", pod.Name, err))
}
return messages, nil
}
@@ -395,21 +412,8 @@ func execCommand(opts execOpts) error {
return errors.WithStack(err)
}
} else {
- var l *provision.LabelSet
- l, err = provision.ServiceLabels(provision.ServiceLabelsOpts{
- App: opts.app,
- ServiceLabelExtendedOpts: provision.ServiceLabelExtendedOpts{
- Prefix: tsuruLabelPrefix,
- Provisioner: provisionerName,
- },
- })
- if err != nil {
- return errors.WithStack(err)
- }
var pods *apiv1.PodList
- pods, err = client.Core().Pods(client.Namespace()).List(metav1.ListOptions{
- LabelSelector: labels.SelectorFromSet(labels.Set(l.ToAppSelector())).String(),
- })
+ pods, err = podsForAppProcess(client, opts.app, "")
if err != nil {
return errors.WithStack(err)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment