operator log

 kubectl logs  percona-xtradb-cluster-operator-d99c748-7zxbt
{"level":"info","ts":1621359567.588477,"logger":"cmd","msg":"Runs on","platform":"kubernetes","version":"v1.21.1"}
{"level":"info","ts":1621359567.588623,"logger":"cmd","msg":"Git commit: 4e80366586138996191aacd7fe1f009eb8ae36c1 Git branch: release-1-8-0 Build time: 2021-04-22T13:22:59Z"}
{"level":"info","ts":1621359567.5886452,"logger":"cmd","msg":"Go Version: go1.16.3"}
{"level":"info","ts":1621359567.5886583,"logger":"cmd","msg":"Go OS/Arch: linux/amd64"}
{"level":"info","ts":1621359567.5886707,"logger":"cmd","msg":"operator-sdk Version: v0.17.1"}
{"level":"info","ts":1621359567.5889187,"logger":"leader","msg":"Trying to become the leader."}
I0518 17:39:28.633459       1 request.go:621] Throttling request took 1.034254953s, request: GET:https://10.96.0.1:443/apis/rbac.authorization.k8s.io/v1beta1?timeout=32s
{"level":"info","ts":1621359568.7602038,"logger":"leader","msg":"No pre-existing lock was found."}
{"level":"info","ts":1621359568.7633045,"logger":"leader","msg":"Became the leader."}
{"level":"info","ts":1621359569.9709015,"logger":"controller-runtime.metrics","msg":"metrics server is starting to listen","addr":":8080"}
{"level":"info","ts":1621359569.9715235,"logger":"cmd","msg":"Registering Components."}
{"level":"info","ts":1621359570.268149,"logger":"controller-runtime.webhook","msg":"registering webhook","path":"/validate-percona-xtradbcluster"}
{"level":"info","ts":1621359570.2681777,"logger":"cmd","msg":"Starting the Cmd."}
{"level":"info","ts":1621359570.2683663,"logger":"controller-runtime.manager","msg":"starting metrics server","path":"/metrics"}
{"level":"info","ts":1621359570.2685757,"logger":"controller-runtime.webhook.webhooks","msg":"starting webhook server"}
{"level":"info","ts":1621359570.2686021,"logger":"controller","msg":"Starting EventSource","controller":"perconaxtradbcluster-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1621359570.2685862,"logger":"controller","msg":"Starting EventSource","controller":"perconaxtradbclusterbackup-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1621359570.2687652,"logger":"controller","msg":"Starting EventSource","controller":"perconaxtradbclusterrestore-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1621359570.2692473,"logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate"}
{"level":"info","ts":1621359570.2694118,"logger":"controller-runtime.webhook","msg":"serving webhook server","host":"","port":9443}
{"level":"info","ts":1621359570.2695482,"logger":"controller-runtime.certwatcher","msg":"Starting certificate watcher"}
{"level":"info","ts":1621359570.3696742,"logger":"controller","msg":"Starting Controller","controller":"perconaxtradbclusterrestore-controller"}
{"level":"info","ts":1621359570.3699248,"logger":"controller","msg":"Starting Controller","controller":"perconaxtradbclusterbackup-controller"}
{"level":"info","ts":1621359570.5693843,"logger":"controller","msg":"Starting Controller","controller":"perconaxtradbcluster-controller"}
{"level":"info","ts":1621359570.5694475,"logger":"controller","msg":"Starting workers","controller":"perconaxtradbcluster-controller","worker count":1}
{"level":"info","ts":1621359570.5707362,"logger":"controller","msg":"Starting workers","controller":"perconaxtradbclusterrestore-controller","worker count":1}
{"level":"info","ts":1621359570.5708342,"logger":"controller","msg":"Starting workers","controller":"perconaxtradbclusterbackup-controller","worker count":1}
I0518 17:41:16.721600       1 request.go:621] Throttling request took 1.041203955s, request: GET:https://10.96.0.1:443/apis/coordination.k8s.io/v1beta1?timeout=32s
{"level":"info","ts":1621359678.5574117,"caller":"zapr/zapr.go:69","msg":"add new job","schedule":"0 4 * * *"}
{"level":"info","ts":1621359678.5575793,"caller":"zapr/zapr.go:69","msg":"add new job","name":"ensure-version/pxc/cluster6","schedule":"0 4 * * *"}
{"level":"info","ts":1621359678.6116567,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359678.6117718,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"error","ts":1621359678.6961565,"caller":"zapr/zapr.go:128","msg":"Update status","error":"send update: Operation cannot be fulfilled on perconaxtradbclusters.pxc.percona.com \"cluster6\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:229\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:531\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90"}
{"level":"info","ts":1621359678.7269235,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359678.7269893,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359683.6114142,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359683.6115243,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359688.6821227,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359688.6822155,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359693.7444913,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359693.744575,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359698.80769,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359698.8078303,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359703.8763614,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359703.8765085,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359708.9436042,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359708.9437246,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359714.0043788,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359714.004482,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359719.0653815,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359719.0654736,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359724.1355453,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359724.1356275,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359729.1999028,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359729.20002,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359734.2715154,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359734.2716234,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359739.3376215,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359739.3376753,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359744.4027953,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359744.4028964,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359744.4907923,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359744.4908893,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359749.4926276,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359749.492726,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359754.5742824,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359754.57437,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359759.6484742,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359759.6485615,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359764.7263243,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359764.726425,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359764.802862,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359764.8029513,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359769.8079526,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359769.8080742,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359774.8944376,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359774.8945484,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359779.972599,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359779.9727042,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359785.0625222,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359785.0626795,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359785.1418498,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359785.1419516,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359790.1404977,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359790.1406035,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359795.232342,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359795.2324154,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359800.3157704,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359800.3159482,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621359805.3988032,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621359805.3988836,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
[mysql] 2021/05/18 18:02:48 packets.go:36: read tcp 192.168.71.193:54910->192.168.66.6:3306: read: connection timed out
{"level":"error","ts":1621360968.5187535,"caller":"zapr/zapr.go:128","msg":"Update status","error":"send update: Operation cannot be fulfilled on perconaxtradbclusters.pxc.percona.com \"cluster6\": StorageError: invalid object, Code: 4, Key: /registry/pxc.percona.com/perconaxtradbclusters/pxc/cluster6, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 3767f964-bb44-4dc6-8f4a-ff72322ea6c8, UID in object meta: ","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:229\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:248\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90"}
{"level":"error","ts":1621360968.5190585,"logger":"controller","msg":"Reconciler error","controller":"perconaxtradbcluster-controller","name":"cluster6","namespace":"pxc","error":"reconcileUsers: manage xtrabackup user: update xtrabackup grant: rollback error: sql: transaction has already been committed or rolled back, transaction failed with: create operator user: invalid connection, tx rollback: invalid connection","errorVerbose":"create operator user: invalid connection, tx rollback: invalid connection\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/pxc/users.(*Manager).Update170XtrabackupUser\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/users/users.go:277\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).manageXtrabackupUser\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:202\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:79\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:246\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371\nrollback error: sql: transaction has already been committed or rolled back, transaction failed with\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/pxc/users.(*Manager).Update170XtrabackupUser.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/users/users.go:264\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/pxc/users.(*Manager).Update170XtrabackupUser\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/users/users.go:277\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).manageXtrabackupUser\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:202\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:79\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:246\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371\nupdate xtrabackup grant\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).manageXtrabackupUser\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:204\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:79\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:246\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371\nmanage xtrabackup user\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:81\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:246\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371\nreconcileUsers\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:248\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1371","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90"}
I0518 18:03:28.157112       1 request.go:621] Throttling request took 1.042021324s, request: GET:https://10.96.0.1:443/apis/pxc.percona.com/v1-2-0?timeout=32s
{"level":"info","ts":1621361009.504205,"caller":"zapr/zapr.go:69","msg":"add new job","schedule":"0 4 * * *"}
{"level":"info","ts":1621361009.5043054,"caller":"zapr/zapr.go:69","msg":"add new job","name":"ensure-version/pxc/cluster7","schedule":"0 4 * * *"}
{"level":"info","ts":1621361009.556918,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361009.55701,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"error","ts":1621361009.6247394,"caller":"zapr/zapr.go:128","msg":"Update status","error":"send update: Operation cannot be fulfilled on perconaxtradbclusters.pxc.percona.com \"cluster7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:229\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:531\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:209\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:188\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90"}
{"level":"info","ts":1621361009.6570218,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361009.6570673,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361014.5606296,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361014.5607107,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361019.6216142,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361019.6217139,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361024.6896005,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361024.6896822,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361029.7589622,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361029.759128,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361034.823406,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361034.823498,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361039.8912308,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361039.8913467,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361044.9540863,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361044.954178,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361050.0395606,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361050.0396976,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361055.1087968,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361055.1089134,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361060.1750317,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361060.1751075,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361065.2445831,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361065.2447357,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361070.3148274,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361070.314954,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361075.380713,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361075.3807924,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361075.4646914,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361075.4647725,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361080.4666765,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361080.4667964,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361085.5556223,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361085.5557196,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361090.6381276,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361090.6382365,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361095.714623,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361095.7147136,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361095.788234,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361095.7883043,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361100.7937925,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361100.793877,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361105.8799238,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361105.8800616,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361110.9551487,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361110.9552279,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361116.0367732,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361116.0368366,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361116.1115782,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361116.111664,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361121.1175795,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361121.117678,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361126.1930244,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361126.193093,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361131.2829933,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361131.2830791,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361136.370927,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361136.3710656,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361136.5346384,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361136.534732,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361141.47218,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361141.4722714,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361146.5646012,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361146.5647156,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361151.6560433,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361151.656158,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361156.7501001,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361156.7502,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361161.8499577,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361161.850056,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361166.9432395,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361166.9432952,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361172.0357904,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361172.0358522,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361177.1347806,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361177.1348844,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361182.2275777,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361182.2276359,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361187.3210866,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361187.3211932,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361192.4158254,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361192.4159625,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361197.518289,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361197.5183947,"caller":"zapr/zapr.go:69","msg":"can't start/continue 'SmartUpdate': waiting for all replicas are ready"}
{"level":"info","ts":1621361202.6219819,"caller":"zapr/zapr.go:69","msg":"statefulSet was changed, run smart update"}
{"level":"info","ts":1621361202.6313882,"caller":"zapr/zapr.go:69","msg":"primary pod","pod name":"cluster7-pxc-0.cluster7-pxc.pxc"}
{"level":"info","ts":1621361202.6314793,"caller":"zapr/zapr.go:69","msg":"apply changes to secondary pod","pod name":"cluster7-pxc-2"}
{"level":"info","ts":1621361202.6315172,"caller":"zapr/zapr.go:69","msg":"pod is already updated","pod name":"cluster7-pxc-2"}
{"level":"info","ts":1621361202.631634,"caller":"zapr/zapr.go:69","msg":"pod is running","pod name":"cluster7-pxc-2"}
{"level":"info","ts":1621361207.6429513,"caller":"zapr/zapr.go:69","msg":"apply changes to secondary pod","pod name":"cluster7-pxc-1"}
{"level":"info","ts":1621361207.643053,"caller":"zapr/zapr.go:69","msg":"pod is already updated","pod name":"cluster7-pxc-1"}
{"level":"info","ts":1621361207.643234,"caller":"zapr/zapr.go:69","msg":"pod is running","pod name":"cluster7-pxc-1"}
{"level":"info","ts":1621361212.6529288,"caller":"zapr/zapr.go:69","msg":"apply changes to primary pod","pod name":"cluster7-pxc-0"}