Skip to content

Instantly share code, notes, and snippets.

@kholisrag
Created June 17, 2022 15:33
Show Gist options
  • Save kholisrag/fa7ed0ef939ae6943c17a7f695b27d9d to your computer and use it in GitHub Desktop.
Save kholisrag/fa7ed0ef939ae6943c17a7f695b27d9d to your computer and use it in GitHub Desktop.
Checking All possible alloc events details when termination already occurs
package main
import (
"encoding/json"
"fmt"
"time"
"github.com/hashicorp/nomad/api"
"github.com/sirupsen/logrus"
)
type allocEventStruct struct {
JobName string `json:"job_name"`
AllocID string `json:"alloc_id"`
Events []map[string]string `json:"events"`
}
func main() {
allocEvents := []allocEventStruct{}
var allocsCount int
nomadCfg := api.DefaultConfig()
nomadCfg.Address = "http://localhost:4646"
nc, err := api.NewClient(nomadCfg)
if err != nil {
logrus.WithField("error_message", err).Fatal("can't connect to nomad")
}
leaderStatus, err := nc.Status().Leader()
if err != nil {
logrus.WithField("error_message", err).Fatal("can't query server status")
}
fmt.Println("Nomad Leader : ", leaderStatus)
listJob, _, err := nc.Jobs().List(&api.QueryOptions{
Prefix: "<Job_Prefix>",
Region: "global",
Params: map[string]string{
"status": "dead",
},
})
if err != nil {
logrus.WithField("error_message", err).Fatal("query job list error")
}
fmt.Printf("Processing, Please Wait.")
for _, job := range listJob {
jobSummary := job.JobSummary.Summary
for _, summaryStatus := range jobSummary {
if summaryStatus.Failed != 0 {
jobAllocs, _, err := nc.Jobs().Allocations(job.ID, true, &api.QueryOptions{})
if err != nil {
logrus.WithField("error_message", err).Fatal("query allocs in %s error", job.Name)
}
for _, alloc := range jobAllocs {
allocInfo, _, err := nc.Allocations().Info(alloc.ID, &api.QueryOptions{})
if err != nil {
logrus.WithField("error_message", err).Fatal("query allocs info %s error", alloc.ID)
}
for _, taskState := range allocInfo.TaskStates {
evenLength := len(taskState.Events) - 1
var evenIndexMarker int
for eventIndex, event := range taskState.Events {
if event.Type == "Terminated" {
evenIndexMarker = eventIndex
}
}
if evenLength > evenIndexMarker {
var eventDetail []map[string]string
for _, event := range taskState.Events {
eventDetail = append(eventDetail, map[string]string{
"type": event.Type,
"time": time.Unix(0, event.Time).Format("2006-01-02T15:04:05Z07:00"),
"details": event.DisplayMessage,
})
}
allocEvents = append(allocEvents, allocEventStruct{
JobName: job.Name,
AllocID: alloc.ID,
Events: eventDetail,
})
allocsCount++
fmt.Printf(".")
}
}
}
}
}
}
prettyAllocEvents, err := json.MarshalIndent(allocEvents, "", " ")
if err != nil {
logrus.WithField("error_message", err).Fatal("error prettify output")
}
fmt.Printf("\nList of Alloc with strange terminated events : \n%s\n", string(prettyAllocEvents))
fmt.Printf("Captured Allocs Count: %v\n", allocsCount)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment