mfojtik

[0.59] 1837540 [BM IPI] Cluster reports upgrade in progress after restoring to previous state after upgrade
       1836270 [BM IPI] etcd containers not started on master after restoring to previous state
[0.53] 1802510 KubeApiErrorsHigh alert is firing sometimes
       1748434 KubeAPIErrorsHigh firing on daily base but at random times
[0.66] 1838410 openshift api server operator is not Available
       1840112 openshift-apiserver not available in fresh env
[0.53] 1748434 KubeAPIErrorsHigh firing on daily base but at random times
       1802510 KubeApiErrorsHigh alert is firing sometimes
[0.98] 1840116 revert "force cert rotation every couple days for development" in 4.6
       1806980 revert "force cert rotation every couple days for development" in 4.5

mfojtik

	StaticPodsDegraded: nodes/ci-op-szpb1k3d-aa909-q5lfg-master-1 pods/kube-apiserver-ci-op-szpb1k3d-aa909-q5lfg-master-1 container=""kube-apiserver"" is terminated: ""Error"" - ""calhost:2379 0  <nil>}. Err :connection error: desc = "transport: Error while dialing dial tcp [::1]:2379: connect: connection refused". Reconnecting...
	StaticPodsDegraded: nodes/ci-op-szpb1k3d-aa909-q5lfg-master-1 pods/kube-apiserver-ci-op-szpb1k3d-aa909-q5lfg-master-1 container=""kube-apiserver"" is terminated: ""Error"" - ""calhost:2379 0  <nil>}. Err :connection error: desc = "transport: Error while dialing dial tcp [::1]:2379: connect: connection refused". Reconnecting...
12:21:21 (1) "openshift-kube-apiserver" TerminationStart Received signal to terminate, becoming unready, but keeping serving
12:21:21 (1) "openshift-kube-apiserver" TerminationPreShutdownHooksFinished All pre-shutdown hooks have been finished
12:22:31 (1) "openshift-kube-apiserver" TerminationMinimalShutdownDurationFinished The minimal shutdown duration of 1m10s

mfojtik

cluster-debug-tools [timeline L|✚ 2] > ./kubectl-dev_tool event -f ../../mfojtik/eventer/events.json --timeline -o wide 
17:20:28 (1) "openshift-kube-storage-version-migrator-operator" OperatorStatusChanged clusteroperator/kube-storage-version-migrator: Progressing changed True -> False
17:20:28 (1) "openshift-kube-storage-version-migrator-operator" OperatorStatusChanged clusteroperator/kube-storage-version-migrator: Degraded changed Unknown -> False
17:20:29 (1) "openshift-etcd-operator" OperatorStatusChanged clusteroperator/etcd: Upgradeable changed Unknown -> True
17:20:29 (1) "openshift-controller-manager-operator" OperatorStatusChanged clusteroperator/openshift-controller-manager: Degraded changed Unknown -> False
17:20:29 (1) "openshift-etcd-operator" OperatorStatusChanged clusteroperator/etcd: Degraded changed Unknown -> False
17:20:29 (1) "openshift-etcd-operator" OperatorStatusChanged clusteroperator/etcd: Degraded changed Unknown -> False
17:20:29 (1) "openshift-kube-controller-manager-operator" Ope

mfojtik

[
  [
    "2020-04-27 17:20:28 +0200 CEST",
    "kube-storage-version-migrator",
    "Degraded",
    "Unknown",
    "set"
  ],
  [
    "2020-04-27 17:20:28 +0200 CEST",

mfojtik

#!/usr/bin/env bash

set -x

mkdir -p /tmp/etcd-metrics

jobName="e2e-azure-serial"
jobVersion="4.5"

# NOTE: Change this to 'stat' if you run on Linux

mfojtik

W0319 12:20:51.342708       1 builder.go:111] Restart triggered because of file /var/run/secrets/serving-cert/tls.crt was created
I0319 12:20:51.342760       1 observer_polling.go:107] Observed file "/var/run/secrets/serving-cert/tls.key" has been created (hash="7e2c4e5a2fed14bccd97a5eb779fe201fe1807969ca0a37e3ebb866547efeee2")
I0319 12:20:51.343115       1 tlsconfig.go:234] Shutting down DynamicServingCertificateController
I0319 12:20:51.343140       1 builder.go:219] server exited
I0319 12:20:51.343564       1 base_controller.go:95] Shutting down NodeController ...
I0319 12:20:51.343586       1 base_controller.go:95] Shutting down ConfigObserver ...
I0319 12:20:51.343599       1 base_controller.go:95] Shutting down ResourceSyncController ...
I0319 12:20:51.343638       1 base_controller.go:95] Shutting down CertRotationController ...
I0319 12:20:51.343655       1 base_controller.go:95] Shutting down BoundSATokenSignerController ...
I0319 12:20:51.343665       1 base_controller.go:95] Shutting down CertRotat

mfojtik

Mar 05 09:51:48.125 I openshift-apiserver OpenShift API started failing: Get https://api.ci-op-gbz7tdm6-77109.origin-ci-int-aws.dev.rhcloud.com:6443/apis/image.openshift.io/v1/namespaces/openshift-apiserver/imagestreams/missing?timeout=3s: context deadline exceeded (Client.Timeout exceeded while awaiting headers)
Mar 05 09:51:54.125 I openshift-apiserver OpenShift API started failing: Get https://api.ci-op-gbz7tdm6-77109.origin-ci-int-aws.dev.rhcloud.com:6443/apis/image.openshift.io/v1/namespaces/openshift-apiserver/imagestreams/missing?timeout=3s: net/http: request canceled (Client.Timeout exceeded while awaiting headers)
Mar 05 09:52:01.125 I openshift-apiserver OpenShift API started failing: Get https://api.ci-op-gbz7tdm6-77109.origin-ci-int-aws.dev.rhcloud.com:6443/apis/image.openshift.io/v1/namespaces/openshift-apiserver/imagestreams/missing?timeout=3s: context deadline exceeded
Mar 05 09:52:07.125 I openshift-apiserver OpenShift API started failing: Get https://api.ci-op-gbz7tdm6-77109.origin-ci-int-aws

mfojtik

base_images:
  base:
    cluster: https://api.ci.openshift.org
    name: "4.4"
    namespace: ocp
    tag: base
binary_build_commands: make build --warn-undefined-variables
build_root:
  image_stream_tag:
    cluster: https://api.ci.openshift.org

mfojtik

library-go [trim-path L|✚ 1] > docker run -it --rm --workdir=/go/src/github.com/openshift/library-go -v `pwd`:/go/src/github.com/openshift/library-go registry.svc.ci.openshift.org/ocp/builder:golang-1.13 make -C alpha-build-machinery update
make: Entering directory `/go/src/github.com/openshift/library-go/alpha-build-machinery'
mkdir -p "./make/"
set -o pipefail; make -j 1 -C "./make/" -f "default.example.mk" --no-print-directory --warn-undefined-variables help 2>&1 | sed 's/\.\(buildDate\|versionFromGit\|commitFromGit\|gitTreeState\)="[^"]*" /.\1="<redacted_for_diff>" /g' | sed -E 's~/.*/(github.com/openshift/library-go/alpha-build-machinery/.*)~/\1~g' | sed '/\/tmp\/tmp./d' | sed '/git checkout -b/d' | sed -E 's~^[<> ]*((\+\+\+|\-\-\-) \./(testing/)?manifests/.*.yaml).*~\1~' | sed -E 's/^(make\[2\]: \*\*\* \[).*: (.*\] Error 1)/\1\2/' | grep -v 'are the same file' | grep -E -v -e '^make\[2\]: Target `.*'"'"' not remade because of errors\.$' | tee "./make/"/"default.example.mk".help.log
The following make ta

mfojtik

library-go [trim-path L|✔] > docker run -it --rm --workdir=/go/src/github.com/openshift/library-go -v `pwd`:/go/src/github.com/openshift/library-go registry.svc.ci.openshift.org/ocp/builder:golang-1.13 make -C alpha-build-machinery update
Unable to find image 'registry.svc.ci.openshift.org/ocp/builder:golang-1.13' locally
golang-1.13: Pulling from ocp/builder
ab5ef0e58194: Already exists
1932728b581d: Pull complete
Digest: sha256:bbd266cc9a9574190a90ea15948374d982968eeb3a58f6b96d42e63698a6f688
Status: Downloaded newer image for registry.svc.ci.openshift.org/ocp/builder:golang-1.13
make: Entering directory `/go/src/github.com/openshift/library-go/alpha-build-machinery'
mkdir -p "./make/"
set -o pipefail; make -j 1 -C "./make/" -f "default.example.mk" --no-print-directory --warn-undefined-variables help 2>&1 | sed 's/\.\(buildDate\|versionFromGit\|commitFromGit\|gitTreeState\)="[^"]*" /.\1="<redacted_for_diff>" /g' | sed -E 's~/.*/(github.com/openshift/library-go/alpha-build-machinery/.*)~/\1~g' | sed '/\/tmp\/