pod_deployment_checker

deployment_check.rb

Thread.abort_on_exception = true
STDOUT.sync = true
STDERR.sync = true

module DeploymentCheck
  def watch_pod_options
    {
    :label_selector   => "app=manageiq",
    :namespace        => my_namespace,
    # :label_selector => "app=manageiq,worker=true"
    }.freeze
  end

  def current_pods
    @_current_pods ||= {}
  end

  def failed_deployments(restart_count = 5)
    current_pods.select { |name, h| h.fetch(:last_state_terminated) && h.fetch(:container_restarts, 0) > 5 }.keys
  end

  def collect
    Thread.new { pod_collector }
  end

  private

  def pod_collector
    resource_version = refresh_all_pods(watch_pod_options)
    pp current_pods
    pp resource_version

    watch_pods(resource_version, watch_pod_options) do |notice|
      pp notice
      save_pod(notice.object)
    end
 end

  def refresh_all_pods(watch_pods_options)
    pods = kube_connection.get_pods(watch_pods_options)
    resource_version = pods.resourceVersion || 0
    pods.each { |p| save_pod(p) }

    resource_version
  rescue => err
    STDERR.puts("[ERROR] #{__method__}, error: #{err.message}\n#{err.backtrace.join("\n")}")
    raise
  end

  def watch_pods(resource_version, watch_pods_options)
    watcher = kube_connection.watch_pods(watch_pods_options.merge(:resource_version => resource_version))
    watcher.each { |notice| yield notice }
  rescue StandardError => err
    STDERR.puts("[ERROR] pod, error: #{err.message}\n#{err.backtrace.join("\n")}")
    raise
  end

  def save_pod(pod)
    name = pod.metadata.name
    current_pods[name] ||= {}
    current_pods[name][:label_name]            = pod.metadata.labels.name
    current_pods[name][:last_state_running]    = !!pod.status.containerStatuses.first.lastState.running
    current_pods[name][:last_state_waiting]    = !!pod.status.containerStatuses.first.lastState.waiting
    current_pods[name][:last_state_terminated] = !!pod.status.containerStatuses.first.lastState.terminated
    current_pods[name][:container_restarts]    = pod.status.containerStatuses.inject(0) { |sum, cs| sum += cs.restartCount if cs.lastState.terminated; sum }

    puts "saved #{name}"
    pp current_pods
  end
end

ContainerOrchestrator.include(DeploymentCheck)

co = ContainerOrchestrator.new
co.collect

output.txt

...saved 1-amazon-cloud-event-catcher-20-cf48f5f4c-tqx4g
{"1-amazon-cloud-event-catcher-20-cf48f5f4c-tqx4g"=>
  {:label_name=>"1-amazon-cloud-event-catcher-20",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>true,
   :container_restarts=>288},
 "1-amazon-cloud-refresh-20-21-22-f9bdf4dc8-4nzkj"=>
  {:label_name=>"1-amazon-cloud-refresh-20-21-22",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>true,
   :container_restarts=>281},
 "1-event-handler-757bc7884d-knvfs"=>
  {:label_name=>"1-event-handler",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-generic-79bb8b8bb5-8ggbg"=>
  {:label_name=>"1-generic",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-generic-79bb8b8bb5-x2fh5"=>
  {:label_name=>"1-generic",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-priority-85b65979dc-kfgc7"=>
  {:label_name=>"1-priority",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-priority-85b65979dc-pxp8w"=>
  {:label_name=>"1-priority",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-reporting-797d455db9-95m24"=>
  {:label_name=>"1-reporting",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-reporting-797d455db9-xgncz"=>
  {:label_name=>"1-reporting",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-schedule-677b5c8fff-vkljj"=>
  {:label_name=>"1-schedule",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-ui-7c78b95d79-fv87b"=>
  {:label_name=>"1-ui",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-ui-7c78b95d79-xlfkv"=>
  {:label_name=>"1-ui",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-web-service-594ccf5bb8-jl8m2"=>
  {:label_name=>"1-web-service",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "1-web-service-594ccf5bb8-vt68s"=>
  {:label_name=>"1-web-service",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "httpd-b89bbdff6-hlxhw"=>
  {:label_name=>"httpd",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "memcached-84987cbdf5-db7tv"=>
  {:label_name=>"memcached",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "orchestrator-bc69fb46c-8zn5j"=>
  {:label_name=>"orchestrator",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0},
 "postgresql-56f89c8fc7-8jkwc"=>
  {:label_name=>"postgresql",
   :last_state_running=>false,
   :last_state_waiting=>false,
   :last_state_terminated=>false,
   :container_restarts=>0}}