new log

gistfile1.txt

++ id -u
+ myuid=1000140000
++ id -g
+ mygid=0
+ set +e
++ getent passwd 1000140000
1000140000
0

+ uidentry=
+ set -e
+ echo 1000140000
+ echo 0
+ echo
+ [[ -z '' ]]
+ [[ -w /etc/passwd ]]
+ echo '1000140000:x:1000140000:0:anonymous uid:/opt/spark:/bin/false'
+ exec /sbin/tini -s -- /usr/bin/spark-operator -enable-metrics=true -metrics-labels=app_type -metrics-port=10254 -metrics-endpoint=/metrics -metrics-prefix= -enable-webhook=true -webhook-svc-namespace=spark -webhook-svc-name=spark-operator-ns-webhook -webhook-config-name=spark-operator-ns-fdp-sparkoperator-webhook-config -webhook-port=8080 -ingress-url-format= -install-crds=true -namespace=spark -controller-threads=10 -resync-interval=30 -logtostderr -v=2
I0319 23:36:42.864521      11 main.go:96] Enabling metrics collecting and exporting to Prometheus
I0319 23:36:42.864635      11 metrics.go:141] Started Metrics server at localhost:10254/metrics
I0319 23:36:42.864662      11 main.go:100] Starting the Spark Operator
I0319 23:36:42.876556      11 crd.go:48] Updating CustomResourceDefinition sparkapplications.sparkoperator.k8s.io
I0319 23:36:43.387993      11 crd.go:48] Updating CustomResourceDefinition scheduledsparkapplications.sparkoperator.k8s.io
I0319 23:36:43.896972      11 controller.go:148] Starting the workers of the SparkApplication controller
I0319 23:36:43.997211      11 controller.go:96] Starting the ScheduledSparkApplication controller
I0319 23:36:44.097307      11 controller.go:102] Starting the workers of the ScheduledSparkApplication controller
I0319 23:36:44.097835      11 webhook.go:114] Starting the Spark pod admission webhook server
I0319 23:36:44.100746      11 webhook.go:234] Creating a MutatingWebhookConfiguration for the Spark pod admission webhook
I0319 23:37:00.467898      11 controller.go:172] SparkApplication spark/spark-pi was added, enqueueing it for submission
I0319 23:37:00.467950      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:00.468090      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:37:00.468174      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038620468114266 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038620468114266 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 10000]
I0319 23:37:00.468256      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636838", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationAdded' SparkApplication spark-pi was added, enqueuing it for submission
I0319 23:37:03.098649      11 webhook.go:136] Serving admission request
I0319 23:37:03.102085      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:37:03.118522      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:37:03.119008      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:03.131323      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:03.496941      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:37:03.497034      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:37:03.497232      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636838", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:37:03.511472      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0}] to [{ spark-pi-1553038620468114266 2019-03-19 23:37:03.496967662 +0000 UTC m=+20.642583071 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    } {SUBMITTED } map[] 1 1}]
I0319 23:37:03.529876      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0} new status: { spark-pi-1553038620468114266 2019-03-19 23:37:03.496967662 +0000 UTC m=+20.642583071 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    } {SUBMITTED } map[] 1 1}
I0319 23:37:03.529934      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:03.532124      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:03.532238      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:03.539347      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    } {SUBMITTED } map[] 1 1}] to [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:37:03.547178      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:03.547288      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    } {SUBMITTED } map[] 1 1} new status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:37:03.547331      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:03.547360      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:03.550232      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:37:03.554661      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:37:03.554728      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:09.382308      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:09.382341      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:09.382383      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:09.382733      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636866", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:37:09.386161      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:37:09.395002      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:37:09.395089      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:37:09.395914      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:09.396353      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:09.396443      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:09.399610      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:37:09.404382      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:37:09.404467      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:13.910523      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:13.910574      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:13.919348      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:37:13.923472      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:37:13.923510      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:18.620956      11 webhook.go:136] Serving admission request
I0319 23:37:18.630679      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038621881-exec-1 added in namespace spark.
I0319 23:37:18.630710      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:18.630750      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:18.631341      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636893", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038621881-exec-1 is pending
I0319 23:37:18.633808      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}]
I0319 23:37:18.636018      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-1 updated in namespace spark.
I0319 23:37:18.636077      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:18.641032      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}
I0319 23:37:18.641117      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:18.641201      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:18.641692      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:18.642043      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636893", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038621881-exec-1 is pending
I0319 23:37:18.644866      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}]
I0319 23:37:18.649633      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-1 updated in namespace spark.
I0319 23:37:18.649661      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:18.655352      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-c7d70455b5bd4a6bb164357da3a21db7 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}
I0319 23:37:18.655394      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:18.655421      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:18.660438      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}]
I0319 23:37:18.660637      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:18.693608      11 webhook.go:136] Serving admission request
I0319 23:37:18.702969      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038621881-exec-2 added in namespace spark.
I0319 23:37:18.703090      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:18.703272      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:18.704720      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636924", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038621881-exec-2 is pending
I0319 23:37:18.708071      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-2 updated in namespace spark.
I0319 23:37:18.709702      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:18.709633      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}]
I0319 23:37:18.717357      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:18.717689      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING] 1 1} new status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}
I0319 23:37:18.717773      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:18.717838      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:18.720977      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-2 updated in namespace spark.
I0319 23:37:18.721048      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:18.721518      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}]
I0319 23:37:18.721649      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:18.721722      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:18.725317      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}]
I0319 23:37:18.725517      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:24.436333      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-2 updated in namespace spark.
I0319 23:37:24.436364      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:24.436387      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:24.436692      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636931", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038621881-exec-2 is running
I0319 23:37:24.445142      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:RUNNING] 1 1}]
I0319 23:37:24.451676      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:PENDING] 1 1} new status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:RUNNING] 1 1}
I0319 23:37:24.451728      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038621881-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:37:24.451751      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:37:24.451769      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:24.452176      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:24.452217      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:24.454915      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:RUNNING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:RUNNING] 1 1}]
I0319 23:37:24.455035      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:24.897615      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-1 updated in namespace spark.
I0319 23:37:24.897651      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:24.897686      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:24.897980      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636965", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038621881-exec-1 is running
I0319 23:37:24.901389      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:RUNNING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}]
I0319 23:37:24.907690      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:PENDING spark-pi-1553038621881-exec-2:RUNNING] 1 1} new status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}
I0319 23:37:24.907739      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038621881-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:37:24.907768      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:37:24.907789      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:24.908499      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:24.908544      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:24.911266      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}]
I0319 23:37:24.911369      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:31.389936      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:31.389989      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:31.390222      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2636993", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:37:31.390357      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:31.393430      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}] to [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {RUNNING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}]
I0319 23:37:31.393540      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:31.393590      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:31.393655      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:37:31.400092      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:31.400115      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:31.406456      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:37:31.406716      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:37:31.406747      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:31.428215      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-2 updated in namespace spark.
I0319 23:37:31.428296      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:31.429581      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-1 updated in namespace spark.
I0319 23:37:31.429700      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:31.440292      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:37:31.447684      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:37:31.447740      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:31.447780      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:31.447854      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:37:31.448372      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:31.451000      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:37:31.453563      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:37:31.463167      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038638319 spark-pi-1553038620468114266 2019-03-19 23:37:03 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038621881-exec-1:RUNNING spark-pi-1553038621881-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:37:31.463212      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:31.463239      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:31.467577      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:37:31.467659      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038651467603063 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038651467603063 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 20000]
I0319 23:37:31.468141      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637009", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:37:31.976035      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-1 updated in namespace spark.
I0319 23:37:31.976086      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:32.532071      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-2 updated in namespace spark.
I0319 23:37:32.532115      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:32.996504      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-1 updated in namespace spark.
I0319 23:37:32.996542      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:33.001238      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038621881-exec-1 deleted in namespace spark.
I0319 23:37:33.001289      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:33.550564      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038621881-exec-2 updated in namespace spark.
I0319 23:37:33.550596      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:33.553892      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038621881-exec-2 deleted in namespace spark.
I0319 23:37:33.553924      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:33.975955      11 webhook.go:136] Serving admission request
I0319 23:37:33.976365      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:37:33.985704      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:37:33.991204      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:34.003286      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:34.363367      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:37:34.363442      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:37:34.363549      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637009", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:37:34.376494      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038651467603063 2019-03-19 23:37:34.363392376 +0000 UTC m=+51.509007808 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    } {SUBMITTED } map[] 1 1}]
I0319 23:37:34.390900      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038651467603063 2019-03-19 23:37:34.363392376 +0000 UTC m=+51.509007808 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    } {SUBMITTED } map[] 1 1}
I0319 23:37:34.390997      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:34.391036      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:34.391365      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:34.394641      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038620468114266 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31364    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:37:34.394766      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:34.394811      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:34.398304      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    } {SUBMITTED } map[] 1 1}] to [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:37:34.408346      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    } {SUBMITTED } map[] 1 1} new status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:37:34.408407      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:34.409079      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:34.409131      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:34.412609      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:37:34.416795      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:37:34.416835      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:40.363182      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:37:40.363238      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:40.363301      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:40.363624      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637045", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:37:40.373366      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:37:40.380946      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:37:40.380994      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:37:40.381016      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:40.381657      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:40.381692      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:40.386239      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:37:40.390611      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:37:40.390652      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:43.910689      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:43.910733      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:43.914432      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:37:43.919024      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:37:43.919068      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:49.735073      11 webhook.go:136] Serving admission request
I0319 23:37:49.744641      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038652880-exec-1 added in namespace spark.
I0319 23:37:49.744716      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:49.744764      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:49.745091      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637070", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038652880-exec-1 is pending
I0319 23:37:49.749388      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-1 updated in namespace spark.
I0319 23:37:49.749429      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:49.750101      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}]
I0319 23:37:49.757199      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}
I0319 23:37:49.757270      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:49.757312      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:49.757442      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:49.757661      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637070", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038652880-exec-1 is pending
I0319 23:37:49.760010      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}]
I0319 23:37:49.762901      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-1 updated in namespace spark.
I0319 23:37:49.762935      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:49.771004      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9600ef7ecb694e85b6a6843573b04c4f spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}
I0319 23:37:49.771057      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:49.771091      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:49.773773      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}]
I0319 23:37:49.773896      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:49.804328      11 webhook.go:136] Serving admission request
I0319 23:37:49.812616      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038652880-exec-2 added in namespace spark.
I0319 23:37:49.812643      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:49.812683      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:49.812921      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637111", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038652880-exec-2 is pending
I0319 23:37:49.815624      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}]
I0319 23:37:49.818870      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-2 updated in namespace spark.
I0319 23:37:49.818900      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:49.823317      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1} new status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}
I0319 23:37:49.823360      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:49.823386      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:49.823663      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637111", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038652880-exec-2 is pending
I0319 23:37:49.824240      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:49.826141      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}]
I0319 23:37:49.837778      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-2 updated in namespace spark.
I0319 23:37:49.837801      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:49.838177      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING] 1 1} new status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}
I0319 23:37:49.838221      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:49.838271      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:49.841037      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}]
I0319 23:37:49.841139      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:55.111592      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-2 updated in namespace spark.
I0319 23:37:55.111624      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:55.111660      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:55.112157      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637118", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038652880-exec-2 is running
I0319 23:37:55.121389      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:RUNNING] 1 1}]
I0319 23:37:55.128706      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:PENDING] 1 1} new status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:RUNNING] 1 1}
I0319 23:37:55.128801      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038652880-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:37:55.128823      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:37:55.128841      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:55.129410      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:55.129462      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:55.132178      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:RUNNING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:RUNNING] 1 1}]
I0319 23:37:55.132316      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:55.801164      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-1 updated in namespace spark.
I0319 23:37:55.801190      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:37:55.801207      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:55.801532      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637152", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038652880-exec-1 is running
I0319 23:37:55.803990      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:RUNNING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1}]
I0319 23:37:55.810857      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:PENDING spark-pi-1553038652880-exec-2:RUNNING] 1 1} new status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1}
I0319 23:37:55.810905      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038652880-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:37:55.810920      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:37:55.810935      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:37:55.811364      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:37:55.811413      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:37:55.814112      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1}] to [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {RUNNING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1}]
I0319 23:37:55.814219      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:02.236676      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:02.236719      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:02.236746      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:38:02.236774      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637178", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:38:02.236747      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:02.243617      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:02.243660      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:02.247157      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:38:02.248472      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:38:02.248495      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:02.273265      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-1 updated in namespace spark.
I0319 23:38:02.273294      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:02.274891      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-2 updated in namespace spark.
I0319 23:38:02.274915      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:02.276423      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:38:02.284888      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:38:02.284938      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:02.284968      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:02.285032      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:38:02.286287      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:02.288581      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:38:02.290828      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:38:02.300142      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038669432 spark-pi-1553038651467603063 2019-03-19 23:37:34 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038652880-exec-1:RUNNING spark-pi-1553038652880-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:38:02.300185      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:02.300216      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:02.304656      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:38:02.304740      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.metrics.namespace=spark.spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038682304680659 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038682304680659 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 30000]
I0319 23:38:02.304864      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637193", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:38:02.890136      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-1 updated in namespace spark.
I0319 23:38:02.890272      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:03.480722      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-2 updated in namespace spark.
I0319 23:38:03.480762      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:03.911205      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-1 updated in namespace spark.
I0319 23:38:03.911272      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:03.915346      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038652880-exec-1 deleted in namespace spark.
I0319 23:38:03.915370      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:04.781795      11 webhook.go:136] Serving admission request
I0319 23:38:04.782190      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:38:04.795397      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:38:04.800950      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:04.813886      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:05.168554      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:38:05.168627      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:38:05.168828      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637193", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:38:05.185356      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038682304680659 2019-03-19 23:38:05.168579219 +0000 UTC m=+82.314194665 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    } {SUBMITTED } map[] 1 1}]
I0319 23:38:05.197150      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038682304680659 2019-03-19 23:38:05.168579219 +0000 UTC m=+82.314194665 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    } {SUBMITTED } map[] 1 1}
I0319 23:38:05.197283      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:05.197328      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:05.197547      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:05.200580      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038651467603063 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31074    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:38:05.200718      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:05.200766      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:05.205163      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    } {SUBMITTED } map[] 1 1}] to [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:38:05.212131      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    } {SUBMITTED } map[] 1 1} new status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:38:05.212172      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:05.213237      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:05.213311      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:05.216238      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:38:05.220608      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:38:05.220671      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:10.303965      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:10.303999      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:10.304025      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:10.304259      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637231", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:38:10.307929      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:38:10.315899      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:38:10.315947      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:38:10.315960      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:10.318190      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:10.318237      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:10.321415      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:38:10.325812      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:38:10.325856      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:12.606562      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038652880-exec-2 updated in namespace spark.
I0319 23:38:12.609830      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038652880-exec-2 deleted in namespace spark.
I0319 23:38:13.910872      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:13.910922      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:13.914492      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:38:13.918543      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:38:13.918584      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:14.292986      11 webhook.go:136] Serving admission request
I0319 23:38:14.304132      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038683617-exec-1 added in namespace spark.
I0319 23:38:14.304162      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:14.304193      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:14.304529      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637254", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038683617-exec-1 is pending
I0319 23:38:14.310348      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}]
I0319 23:38:14.311190      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-1 updated in namespace spark.
I0319 23:38:14.311219      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:14.318419      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}
I0319 23:38:14.318461      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:14.318489      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:14.318780      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637254", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038683617-exec-1 is pending
I0319 23:38:14.318993      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:14.321739      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}]
I0319 23:38:14.326348      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-1 updated in namespace spark.
I0319 23:38:14.326434      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:14.332349      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-e528fcbc848f4fb2bd6a2936bca5a522 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}
I0319 23:38:14.332390      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:14.332425      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:14.335123      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}]
I0319 23:38:14.335278      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:14.349647      11 webhook.go:136] Serving admission request
I0319 23:38:14.358221      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038683617-exec-2 added in namespace spark.
I0319 23:38:14.358328      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:14.358364      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:14.358741      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637275", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038683617-exec-2 is pending
I0319 23:38:14.361406      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}]
I0319 23:38:14.364457      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-2 updated in namespace spark.
I0319 23:38:14.364481      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:14.369310      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1} new status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}
I0319 23:38:14.369352      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:14.369381      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:14.369620      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637275", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038683617-exec-2 is pending
I0319 23:38:14.369784      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:14.372106      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}]
I0319 23:38:14.379856      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-2 updated in namespace spark.
I0319 23:38:14.379888      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:14.382742      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING] 1 1} new status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}
I0319 23:38:14.382784      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:14.382810      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:14.385450      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}]
I0319 23:38:14.385548      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:19.072932      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-2 updated in namespace spark.
I0319 23:38:19.072964      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:19.072988      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:19.073268      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637281", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038683617-exec-2 is running
I0319 23:38:19.080244      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:RUNNING] 1 1}]
I0319 23:38:19.086682      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:PENDING] 1 1} new status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:RUNNING] 1 1}
I0319 23:38:19.086736      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038683617-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:38:19.086754      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:38:19.086780      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:19.086977      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:19.087009      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:19.089646      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:RUNNING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:RUNNING] 1 1}]
I0319 23:38:19.089765      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:20.196541      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-1 updated in namespace spark.
I0319 23:38:20.196657      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:20.196752      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:20.197217      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637310", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038683617-exec-1 is running
I0319 23:38:20.201220      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:RUNNING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}]
I0319 23:38:20.209127      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:PENDING spark-pi-1553038683617-exec-2:RUNNING] 1 1} new status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}
I0319 23:38:20.209186      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038683617-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:38:20.209237      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:38:20.209308      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:20.209639      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:20.209673      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:20.212511      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}]
I0319 23:38:20.212636      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:43.911102      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:43.911196      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:43.920804      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}] to [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {RUNNING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}]
I0319 23:38:43.920961      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:44.884084      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:44.884141      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637395", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:38:44.884160      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:44.884178      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:44.884293      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:38:44.890864      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:44.890903      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:44.894310      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:38:44.895118      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:38:44.895151      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:44.914188      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-2 updated in namespace spark.
I0319 23:38:44.914216      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:44.916239      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-1 updated in namespace spark.
I0319 23:38:44.916413      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:44.922076      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:38:44.929164      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:38:44.929271      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:44.929423      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:44.929530      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:38:44.929831      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:44.932141      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:38:44.934495      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:38:44.943943      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038694057 spark-pi-1553038682304680659 2019-03-19 23:38:05 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038683617-exec-1:RUNNING spark-pi-1553038683617-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:38:44.944013      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:44.944059      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:44.948302      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:38:44.948390      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038724948330447 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038724948330447 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 10000]
I0319 23:38:44.948517      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637411", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:38:45.530402      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-1 updated in namespace spark.
I0319 23:38:45.530439      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:46.424890      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-2 updated in namespace spark.
I0319 23:38:46.424928      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:46.556376      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-1 updated in namespace spark.
I0319 23:38:46.556420      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:46.559824      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038683617-exec-1 deleted in namespace spark.
I0319 23:38:46.559887      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:47.439890      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038683617-exec-2 updated in namespace spark.
I0319 23:38:47.439922      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:47.444210      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038683617-exec-2 deleted in namespace spark.
I0319 23:38:47.444242      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:47.464783      11 webhook.go:136] Serving admission request
I0319 23:38:47.465174      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:38:47.473778      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:38:47.478855      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:47.491761      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:47.799963      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:38:47.800033      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:38:47.800178      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637411", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:38:47.812532      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038724948330447 2019-03-19 23:38:47.799986991 +0000 UTC m=+124.945602429 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    } {SUBMITTED } map[] 1 1}]
I0319 23:38:47.823688      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038724948330447 2019-03-19 23:38:47.799986991 +0000 UTC m=+124.945602429 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    } {SUBMITTED } map[] 1 1}
I0319 23:38:47.823740      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:47.823778      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:47.824915      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:47.827078      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038682304680659 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30933    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:38:47.827194      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:47.827236      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:47.830477      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    } {SUBMITTED } map[] 1 1}] to [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:38:47.838108      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:47.838872      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    } {SUBMITTED } map[] 1 1} new status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:38:47.838910      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:47.838936      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:47.841709      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:38:47.845546      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:38:47.845585      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:52.765425      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:38:52.765462      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:52.765615      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:52.765930      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637443", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:38:52.768614      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:38:52.776277      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:38:52.776326      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:38:52.776351      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:52.777171      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:52.777215      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:52.781314      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:38:52.785348      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:38:52.785390      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:56.882407      11 webhook.go:136] Serving admission request
I0319 23:38:56.893651      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038726348-exec-1 added in namespace spark.
I0319 23:38:56.893677      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:56.893713      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:56.894008      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637466", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038726348-exec-1 is pending
I0319 23:38:56.898280      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-1 updated in namespace spark.
I0319 23:38:56.898305      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:56.903075      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}]
I0319 23:38:56.909513      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}
I0319 23:38:56.909571      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:56.909603      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:56.909894      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637466", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038726348-exec-1 is pending
I0319 23:38:56.910041      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:56.913138      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-1 updated in namespace spark.
I0319 23:38:56.913159      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:56.913207      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}]
I0319 23:38:56.922134      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ac7bb530a6504beaaf1a67c3c7cd2082 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}
I0319 23:38:56.922178      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:56.922201      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:56.924908      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}]
I0319 23:38:56.925009      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:56.992217      11 webhook.go:136] Serving admission request
I0319 23:38:57.001480      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038726348-exec-2 added in namespace spark.
I0319 23:38:57.001510      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:57.001547      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:57.001863      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637482", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038726348-exec-2 is pending
I0319 23:38:57.005731      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-2 updated in namespace spark.
I0319 23:38:57.005757      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:57.005853      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}]
I0319 23:38:57.013059      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1} new status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}
I0319 23:38:57.013122      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:57.013152      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:57.013546      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637482", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038726348-exec-2 is pending
I0319 23:38:57.013922      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:38:57.016191      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}]
I0319 23:38:57.021994      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-2 updated in namespace spark.
I0319 23:38:57.022025      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:38:57.027617      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING] 1 1} new status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}
I0319 23:38:57.027658      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:38:57.027690      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:38:57.030345      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}]
I0319 23:38:57.030444      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:01.726129      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-1 updated in namespace spark.
I0319 23:39:01.726163      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:01.726235      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:01.726623      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637488", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038726348-exec-1 is running
I0319 23:39:01.729635      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:PENDING] 1 1}]
I0319 23:39:01.736984      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:PENDING spark-pi-1553038726348-exec-2:PENDING] 1 1} new status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:PENDING] 1 1}
I0319 23:39:01.737081      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038726348-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:39:01.737145      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:39:01.737162      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:01.737437      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:01.737473      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:01.740351      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:PENDING] 1 1}]
I0319 23:39:01.740553      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:02.920752      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-2 updated in namespace spark.
I0319 23:39:02.920790      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:02.920823      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:02.921088      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637511", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038726348-exec-2 is running
I0319 23:39:02.923728      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:PENDING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}]
I0319 23:39:02.930926      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:PENDING] 1 1} new status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}
I0319 23:39:02.930977      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038726348-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:39:02.931002      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:39:02.931016      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:02.931590      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:02.931715      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:02.934518      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}]
I0319 23:39:02.934651      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:13.911277      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:13.911461      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:13.920228      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}]
I0319 23:39:13.920437      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:15.708347      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:15.708439      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:15.708575      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637560", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:39:15.709015      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:15.711782      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}] to [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {RUNNING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}]
I0319 23:39:15.711969      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:15.712048      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:15.712120      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:39:15.718160      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:15.718189      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:15.721065      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:39:15.722913      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:39:15.723006      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:15.760573      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038724948330447 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:39:15.761460      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-1 updated in namespace spark.
I0319 23:39:15.763605      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:15.763685      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-2 updated in namespace spark.
I0319 23:39:15.763732      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:15.770535      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038724948330447 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:39:15.770631      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:15.770713      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:15.770780      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:39:15.773723      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:39:15.773985      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:15.776108      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038724948330447 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:39:15.785734      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038736634 spark-pi-1553038724948330447 2019-03-19 23:38:47 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038726348-exec-1:RUNNING spark-pi-1553038726348-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038724948330447 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:39:15.785780      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:15.785807      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:15.789987      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:39:15.790071      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038755790019495 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038755790019495 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 20000]
I0319 23:39:15.790299      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637575", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:39:16.922186      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-1 updated in namespace spark.
I0319 23:39:16.922223      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:17.091366      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-2 updated in namespace spark.
I0319 23:39:17.091449      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:17.945588      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-1 updated in namespace spark.
I0319 23:39:17.945629      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:17.949996      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038726348-exec-1 deleted in namespace spark.
I0319 23:39:17.950018      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:18.370464      11 webhook.go:136] Serving admission request
I0319 23:39:18.370854      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:39:18.382993      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:39:18.388917      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:18.402817      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:18.709631      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:39:18.709704      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:39:18.709989      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637575", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:39:18.722201      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038724948330447 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038755790019495 2019-03-19 23:39:18.709656808 +0000 UTC m=+155.855272262 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    } {SUBMITTED } map[] 1 1}]
I0319 23:39:18.731199      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:18.731567      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038724948330447 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30772    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038755790019495 2019-03-19 23:39:18.709656808 +0000 UTC m=+155.855272262 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    } {SUBMITTED } map[] 1 1}
I0319 23:39:18.731631      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:18.731664      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:18.737212      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    } {SUBMITTED } map[] 1 1}] to [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:39:18.744998      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:18.745661      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    } {SUBMITTED } map[] 1 1} new status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:39:18.745698      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:18.745724      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:18.748703      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:39:18.752778      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:39:18.752816      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:22.607724      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038726348-exec-2 updated in namespace spark.
I0319 23:39:22.610888      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038726348-exec-2 deleted in namespace spark.
I0319 23:39:24.035392      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:24.035448      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:24.035583      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:24.036145      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637603", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:39:24.044830      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:39:24.052594      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:39:24.052661      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:39:24.052679      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:24.053468      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:24.053568      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:24.056619      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:39:24.061339      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:39:24.061393      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:33.231740      11 webhook.go:136] Serving admission request
I0319 23:39:33.242308      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038757191-exec-1 added in namespace spark.
I0319 23:39:33.242382      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:33.242478      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:33.242851      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637633", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038757191-exec-1 is pending
I0319 23:39:33.246857      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}]
I0319 23:39:33.246922      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-1 updated in namespace spark.
I0319 23:39:33.247029      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:33.255142      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:33.255517      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-9237e2be7cd8497d88ca648de5b928df spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}
I0319 23:39:33.255594      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:33.255680      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:33.258472      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}]
I0319 23:39:33.258646      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:33.260636      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-1 updated in namespace spark.
I0319 23:39:33.260686      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:33.260752      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:33.263539      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}]
I0319 23:39:33.263725      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:33.327110      11 webhook.go:136] Serving admission request
I0319 23:39:33.338439      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038757191-exec-2 added in namespace spark.
I0319 23:39:33.342795      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:33.342878      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:33.343402      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637663", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038757191-exec-2 is pending
I0319 23:39:33.344006      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-2 updated in namespace spark.
I0319 23:39:33.344075      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:33.354874      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING spark-pi-1553038757191-exec-2:PENDING] 1 1}]
I0319 23:39:33.355990      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-2 updated in namespace spark.
I0319 23:39:33.356055      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:33.366837      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:33.367105      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING] 1 1} new status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING spark-pi-1553038757191-exec-2:PENDING] 1 1}
I0319 23:39:33.367244      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:33.367342      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:33.378643      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING spark-pi-1553038757191-exec-2:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING spark-pi-1553038757191-exec-2:PENDING] 1 1}]
I0319 23:39:33.378799      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:39.283358      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-1 updated in namespace spark.
I0319 23:39:39.283394      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:39.283433      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:39.283711      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637669", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038757191-exec-1 is running
I0319 23:39:39.290976      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING spark-pi-1553038757191-exec-2:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:PENDING] 1 1}]
I0319 23:39:39.298119      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:PENDING spark-pi-1553038757191-exec-2:PENDING] 1 1} new status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:PENDING] 1 1}
I0319 23:39:39.298187      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038757191-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:39:39.298209      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:39:39.298231      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:39.298695      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:39.298741      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:39.301561      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:PENDING] 1 1}]
I0319 23:39:39.301686      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:39.992034      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-2 updated in namespace spark.
I0319 23:39:39.992071      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:39.992112      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:39.992452      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637695", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038757191-exec-2 is running
I0319 23:39:39.995522      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:PENDING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}]
I0319 23:39:40.002152      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:PENDING] 1 1} new status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}
I0319 23:39:40.002199      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038757191-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:39:40.002216      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:39:40.002226      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:40.003080      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:40.003111      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:40.006034      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}]
I0319 23:39:40.006153      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:43.911441      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:43.911502      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:43.914705      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}]
I0319 23:39:43.914828      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:46.564926      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:46.564988      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:46.565227      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637724", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:39:46.565792      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:46.568562      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}] to [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {RUNNING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}]
I0319 23:39:46.568755      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:46.568842      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:46.568901      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:39:46.575410      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:46.575493      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:46.578334      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:39:46.585820      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:39:46.585978      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:46.607475      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-2 updated in namespace spark.
I0319 23:39:46.607512      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:46.610696      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038755790019495 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:39:46.611450      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-1 updated in namespace spark.
I0319 23:39:46.611722      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:46.621350      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038755790019495 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:39:46.621434      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:46.621515      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:46.621635      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:39:46.622077      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:46.628471      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:39:46.631229      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038755790019495 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:39:46.646690      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038772886 spark-pi-1553038755790019495 2019-03-19 23:39:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038757191-exec-1:RUNNING spark-pi-1553038757191-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038755790019495 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:39:46.646775      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:46.646805      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:46.651560      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:39:46.651761      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038786651686537 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038786651686537 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 30000]
I0319 23:39:46.652602      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637739", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:39:47.376080      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-1 updated in namespace spark.
I0319 23:39:47.376186      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:48.101023      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-2 updated in namespace spark.
I0319 23:39:48.101154      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:48.395848      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-1 updated in namespace spark.
I0319 23:39:48.395883      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:48.400471      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038757191-exec-1 deleted in namespace spark.
I0319 23:39:48.400495      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:49.138611      11 webhook.go:136] Serving admission request
I0319 23:39:49.139063      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:39:49.148950      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:39:49.154292      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:49.167309      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:49.537592      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:39:49.537665      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:39:49.537882      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637739", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:39:49.550821      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038755790019495 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038786651686537 2019-03-19 23:39:49.537618377 +0000 UTC m=+186.683233814 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    } {SUBMITTED } map[] 1 1}]
I0319 23:39:49.562949      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:49.564305      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038755790019495 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30372    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038786651686537 2019-03-19 23:39:49.537618377 +0000 UTC m=+186.683233814 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    } {SUBMITTED } map[] 1 1}
I0319 23:39:49.564371      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:49.564407      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:49.567596      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    } {SUBMITTED } map[] 1 1}] to [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:39:49.574131      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:49.575418      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    } {SUBMITTED } map[] 1 1} new status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:39:49.575467      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:49.575495      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:49.578568      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:39:49.583889      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:39:49.583935      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:53.155210      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038757191-exec-2 updated in namespace spark.
I0319 23:39:53.162198      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038757191-exec-2 deleted in namespace spark.
I0319 23:39:54.950183      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:39:54.950431      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:39:54.950581      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:54.951007      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637771", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:39:54.955446      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:39:54.963625      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:39:54.963679      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:39:54.963692      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:39:54.964050      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:39:54.964097      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:39:54.967682      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:39:54.972195      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:39:54.972278      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:04.243897      11 webhook.go:136] Serving admission request
I0319 23:40:04.256472      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038788031-exec-1 added in namespace spark.
I0319 23:40:04.256499      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:04.256528      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:04.256839      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637801", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038788031-exec-1 is pending
I0319 23:40:04.262459      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-1 updated in namespace spark.
I0319 23:40:04.262482      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:04.264552      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}]
I0319 23:40:04.272398      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}
I0319 23:40:04.272442      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:04.272472      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:04.272612      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:40:04.272670      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637801", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038788031-exec-1 is pending
I0319 23:40:04.274569      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-1 updated in namespace spark.
I0319 23:40:04.274596      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:04.275583      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}]
I0319 23:40:04.284991      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-ad61cbea9ff44dc4abdfb2c1fbaded06 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}
I0319 23:40:04.285037      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:04.285068      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:04.287708      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}]
I0319 23:40:04.287829      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:04.335783      11 webhook.go:136] Serving admission request
I0319 23:40:04.345485      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038788031-exec-2 added in namespace spark.
I0319 23:40:04.345511      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:04.345539      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:04.345825      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637835", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038788031-exec-2 is pending
I0319 23:40:04.348554      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}]
I0319 23:40:04.352455      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-2 updated in namespace spark.
I0319 23:40:04.352479      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:04.356780      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1} new status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}
I0319 23:40:04.356830      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:04.356875      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:04.357048      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637835", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038788031-exec-2 is pending
I0319 23:40:04.358263      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:40:04.359739      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}]
I0319 23:40:04.364593      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-2 updated in namespace spark.
I0319 23:40:04.364624      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:04.370274      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING] 1 1} new status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}
I0319 23:40:04.370314      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:04.370340      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:04.373187      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}]
I0319 23:40:04.373311      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:09.332645      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-1 updated in namespace spark.
I0319 23:40:09.332689      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:09.332727      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:09.333068      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637839", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038788031-exec-1 is running
I0319 23:40:09.335666      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:PENDING] 1 1}]
I0319 23:40:09.342369      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:PENDING spark-pi-1553038788031-exec-2:PENDING] 1 1} new status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:PENDING] 1 1}
I0319 23:40:09.342417      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038788031-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:40:09.342433      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:40:09.342452      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:09.342943      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:40:09.342975      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:09.345675      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:PENDING] 1 1}]
I0319 23:40:09.345813      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:10.679438      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-2 updated in namespace spark.
I0319 23:40:10.679476      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:40:10.679521      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:10.680061      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2637862", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038788031-exec-2 is running
I0319 23:40:10.682473      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:PENDING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}]
I0319 23:40:10.690466      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:PENDING] 1 1} new status: {spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}
I0319 23:40:10.690515      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038788031-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:40:10.690539      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:40:10.690558      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:10.691316      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:40:10.691369      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:10.694095      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}]
I0319 23:40:10.694221      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:13.911631      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:40:13.911698      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:13.915192      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}]
I0319 23:40:13.915351      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:40:43.911821      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:40:43.911880      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:40:43.920010      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}] to [{spark-application-1553038803914 spark-pi-1553038786651686537 2019-03-19 23:39:49 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31390    spark-pi-driver} {RUNNING } map[spark-pi-1553038788031-exec-1:RUNNING spark-pi-1553038788031-exec-2:RUNNING] 1 1}]
I0319 23:40:43.920147      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:00.839002      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:41:00.850513      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:00.862092      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:41:00.865803      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"e473e89f-4a9f-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638030", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationDeleted' SparkApplication spark-pi was deleted
I0319 23:41:00.865838      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:00.866204      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:41:00.886026      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-1 updated in namespace spark.
I0319 23:41:00.898077      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-2 updated in namespace spark.
I0319 23:41:01.983085      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-1 updated in namespace spark.
I0319 23:41:02.333351      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-2 updated in namespace spark.
I0319 23:41:03.347364      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-2 updated in namespace spark.
I0319 23:41:03.350677      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038788031-exec-2 deleted in namespace spark.
I0319 23:41:09.500100      11 controller.go:172] SparkApplication spark/spark-pi was added, enqueueing it for submission
I0319 23:41:09.500154      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:09.500213      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:41:09.500339      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038869500240940 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038869500240940 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 10000]
I0319 23:41:09.500491      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638085", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationAdded' SparkApplication spark-pi was added, enqueuing it for submission
I0319 23:41:11.931518      11 webhook.go:136] Serving admission request
I0319 23:41:11.933656      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:41:11.945542      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:41:11.951330      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:11.962789      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:12.302668      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:41:12.302746      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:41:12.302933      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638085", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:41:12.315233      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0}] to [{ spark-pi-1553038869500240940 2019-03-19 23:41:12.302692215 +0000 UTC m=+269.448307668 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    } {SUBMITTED } map[] 1 1}]
I0319 23:41:12.329680      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:12.330076      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0} new status: { spark-pi-1553038869500240940 2019-03-19 23:41:12.302692215 +0000 UTC m=+269.448307668 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    } {SUBMITTED } map[] 1 1}
I0319 23:41:12.330123      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:12.330151      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:12.332898      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    } {SUBMITTED } map[] 1 1}] to [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:41:12.340593      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    } {SUBMITTED } map[] 1 1} new status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:41:12.340637      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:12.341765      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:12.341814      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:12.344692      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:41:12.348689      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:41:12.348732      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:13.155361      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038788031-exec-1 updated in namespace spark.
I0319 23:41:13.158878      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038788031-exec-1 deleted in namespace spark.
I0319 23:41:13.911979      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:13.912040      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:13.915497      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:41:13.920102      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:41:13.920146      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:17.508859      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:17.508891      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:17.508914      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:17.509240      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638110", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:41:17.513911      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:41:17.520985      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:41:17.521039      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:41:17.521061      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:17.521284      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:17.521339      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:17.524172      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:41:17.528157      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:41:17.528211      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:26.862749      11 webhook.go:136] Serving admission request
I0319 23:41:26.872592      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038870784-exec-1 added in namespace spark.
I0319 23:41:26.872619      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:26.872642      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:26.872936      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638139", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038870784-exec-1 is pending
I0319 23:41:26.877162      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-1 updated in namespace spark.
I0319 23:41:26.877190      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:26.883627      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}]
I0319 23:41:26.891765      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-1 updated in namespace spark.
I0319 23:41:26.891795      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:26.891784      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}
I0319 23:41:26.891824      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:26.891853      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:26.892273      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638139", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038870784-exec-1 is pending
I0319 23:41:26.892400      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:26.895407      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}]
I0319 23:41:26.904715      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-748892851222495a891a05ad019a8571 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}
I0319 23:41:26.904759      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:26.904809      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:26.907624      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}]
I0319 23:41:26.907719      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:26.919684      11 webhook.go:136] Serving admission request
I0319 23:41:26.929828      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038870784-exec-2 added in namespace spark.
I0319 23:41:26.929854      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:26.929887      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:26.930155      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638175", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038870784-exec-2 is pending
I0319 23:41:26.933300      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}]
I0319 23:41:26.934312      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-2 updated in namespace spark.
I0319 23:41:26.934380      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:26.941069      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1} new status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}
I0319 23:41:26.941116      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:26.941139      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:26.941312      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638175", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038870784-exec-2 is pending
I0319 23:41:26.941521      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:26.943925      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}]
I0319 23:41:26.946420      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-2 updated in namespace spark.
I0319 23:41:26.946452      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:26.953294      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING] 1 1} new status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}
I0319 23:41:26.953336      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:26.953364      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:26.956054      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}]
I0319 23:41:26.956150      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:31.516792      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-1 updated in namespace spark.
I0319 23:41:31.516825      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:31.516874      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:31.517466      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638182", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038870784-exec-1 is running
I0319 23:41:31.520133      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:PENDING] 1 1}]
I0319 23:41:31.527754      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:PENDING spark-pi-1553038870784-exec-2:PENDING] 1 1} new status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:PENDING] 1 1}
I0319 23:41:31.527805      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038870784-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:41:31.527874      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:41:31.527926      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:31.528116      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:31.528156      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:31.530818      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:PENDING] 1 1}]
I0319 23:41:31.530940      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:32.320095      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-2 updated in namespace spark.
I0319 23:41:32.320129      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:32.320171      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:32.320663      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638205", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038870784-exec-2 is running
I0319 23:41:32.323647      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:PENDING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}]
I0319 23:41:32.330002      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:PENDING] 1 1} new status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}
I0319 23:41:32.330054      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038870784-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:41:32.330075      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:41:32.330096      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:32.330860      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:32.330909      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:32.333593      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}]
I0319 23:41:32.333722      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:40.384937      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:40.384982      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:40.385151      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638247", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:41:40.385653      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:40.388034      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}] to [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {RUNNING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}]
I0319 23:41:40.388165      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:40.388213      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:40.388270      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:41:40.393797      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:40.393823      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:40.397024      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:41:40.397956      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:41:40.397979      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:40.419468      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038869500240940 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:41:40.426195      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-1 updated in namespace spark.
I0319 23:41:40.426233      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:40.426878      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-2 updated in namespace spark.
I0319 23:41:40.426957      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:40.431639      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:40.432155      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038886494 spark-pi-1553038869500240940 2019-03-19 23:41:12 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038870784-exec-1:RUNNING spark-pi-1553038870784-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038869500240940 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:41:40.432318      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:40.432424      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:40.444063      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:41:40.444237      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038900444095048 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038900444095048 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 20000]
I0319 23:41:40.444288      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638262", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:41:41.428170      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-2 updated in namespace spark.
I0319 23:41:41.428209      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:41.644679      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-1 updated in namespace spark.
I0319 23:41:41.644718      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:42.669411      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-1 updated in namespace spark.
I0319 23:41:42.669440      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:42.673795      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038870784-exec-1 deleted in namespace spark.
I0319 23:41:42.673817      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:43.029472      11 webhook.go:136] Serving admission request
I0319 23:41:43.029854      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:41:43.052985      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:41:43.057678      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:43.071765      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:43.424987      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:41:43.425073      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:41:43.425174      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638262", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:41:43.438551      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038869500240940 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038900444095048 2019-03-19 23:41:43.42501404 +0000 UTC m=+300.570629724 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    } {SUBMITTED } map[] 1 1}]
I0319 23:41:43.452337      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:43.452429      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038869500240940 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30025    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038900444095048 2019-03-19 23:41:43.42501404 +0000 UTC m=+300.570629724 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    } {SUBMITTED } map[] 1 1}
I0319 23:41:43.452489      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:43.452520      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:43.458730      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    } {SUBMITTED } map[] 1 1}] to [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:41:43.465748      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    } {SUBMITTED } map[] 1 1} new status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:41:43.465796      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:43.469079      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:43.469127      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:43.472144      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:41:43.476450      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:41:43.476500      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:43.912148      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:43.912198      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:43.916095      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:41:43.920985      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:41:43.921025      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:49.549115      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:41:49.549152      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:49.549196      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:49.549509      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638295", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:41:49.560747      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:41:49.577682      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:49.577746      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:41:49.577805      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:41:49.577835      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:49.577873      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:49.581131      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:41:49.586523      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:41:49.586567      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:53.154404      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038870784-exec-2 updated in namespace spark.
I0319 23:41:53.157671      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038870784-exec-2 deleted in namespace spark.
I0319 23:41:53.423521      11 webhook.go:136] Serving admission request
I0319 23:41:53.638621      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038901758-exec-1 added in namespace spark.
I0319 23:41:53.638657      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:53.638706      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:53.638986      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638323", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038901758-exec-1 is pending
I0319 23:41:53.644579      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-1 updated in namespace spark.
I0319 23:41:53.644610      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:53.648916      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}]
I0319 23:41:53.657219      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:53.657609      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-7b2f5f41581d418db906df93e783ba95 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}
I0319 23:41:53.657657      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:53.657690      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:53.660457      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-1 updated in namespace spark.
I0319 23:41:53.660620      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:53.661305      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}]
I0319 23:41:53.661419      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:53.661461      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:53.664968      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}]
I0319 23:41:53.665063      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:53.682434      11 webhook.go:136] Serving admission request
I0319 23:41:53.692174      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038901758-exec-2 added in namespace spark.
I0319 23:41:53.692208      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:53.692239      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:53.692602      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638349", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038901758-exec-2 is pending
I0319 23:41:53.695650      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}]
I0319 23:41:53.696713      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-2 updated in namespace spark.
I0319 23:41:53.696737      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:53.704380      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1} new status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}
I0319 23:41:53.704426      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:53.704451      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:53.704638      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638349", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038901758-exec-2 is pending
I0319 23:41:53.705234      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:53.709342      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}]
I0319 23:41:53.711101      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-2 updated in namespace spark.
I0319 23:41:53.711130      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:53.718997      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING] 1 1} new status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}
I0319 23:41:53.719058      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:53.719089      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:53.721812      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}]
I0319 23:41:53.721914      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:58.860158      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-1 updated in namespace spark.
I0319 23:41:58.860212      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:58.860322      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:58.860695      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638356", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038901758-exec-1 is running
I0319 23:41:58.864131      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:PENDING] 1 1}]
I0319 23:41:58.870631      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:PENDING spark-pi-1553038901758-exec-2:PENDING] 1 1} new status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:PENDING] 1 1}
I0319 23:41:58.870688      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038901758-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:41:58.870709      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:41:58.870724      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:58.871379      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:58.871433      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:58.874121      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:PENDING] 1 1}]
I0319 23:41:58.874282      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:58.995574      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-2 updated in namespace spark.
I0319 23:41:58.995609      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:41:58.995649      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:58.995959      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638382", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038901758-exec-2 is running
I0319 23:41:58.999768      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:PENDING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}]
I0319 23:41:59.006769      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:PENDING] 1 1} new status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}
I0319 23:41:59.006821      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038901758-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:41:59.006835      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:41:59.006846      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:41:59.007338      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:41:59.007397      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:41:59.010169      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}]
I0319 23:41:59.010286      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:11.178859      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:11.178912      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:11.179060      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638427", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:42:11.179457      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:11.182036      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}] to [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {RUNNING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}]
I0319 23:42:11.182184      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:11.182285      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:11.182351      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:42:11.188707      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:42:11.188770      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:11.191682      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:42:11.192516      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:42:11.192574      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:11.227236      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-2 updated in namespace spark.
I0319 23:42:11.230631      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:11.230797      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:42:11.231895      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-1 updated in namespace spark.
I0319 23:42:11.231952      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:11.238874      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:42:11.238999      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:11.239092      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:11.239161      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:42:11.240227      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:11.241612      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:42:11.244408      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:42:11.254085      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038913193 spark-pi-1553038900444095048 2019-03-19 23:41:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {INVALIDATING } map[spark-pi-1553038901758-exec-1:RUNNING spark-pi-1553038901758-exec-2:RUNNING] 1 1} new status: { spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:42:11.254150      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:11.254183      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:11.258548      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:42:11.258648      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038931258585742 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553038931258585742 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 30000]
I0319 23:42:11.259147      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638443", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:42:12.039037      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-1 updated in namespace spark.
I0319 23:42:12.039082      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:12.151938      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-2 updated in namespace spark.
I0319 23:42:12.151978      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:13.170812      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-2 updated in namespace spark.
I0319 23:42:13.170848      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:13.173729      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038901758-exec-2 deleted in namespace spark.
I0319 23:42:13.173754      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:13.912479      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:13.971201      11 webhook.go:136] Serving admission request
I0319 23:42:13.971630      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:42:13.981550      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:42:13.986560      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:42:14.002468      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:42:14.310505      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:42:14.310581      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:42:14.310798      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638443", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:42:14.323576      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038931258585742 2019-03-19 23:42:14.310532958 +0000 UTC m=+331.456148425 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    } {SUBMITTED } map[] 1 1}]
I0319 23:42:14.332166      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553038931258585742 2019-03-19 23:42:14.310532958 +0000 UTC m=+331.456148425 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    } {SUBMITTED } map[] 1 1}
I0319 23:42:14.332215      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:14.332264      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:14.334940      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:14.336953      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553038900444095048 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32007    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:42:14.337052      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:14.337082      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:14.339782      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    } {SUBMITTED } map[] 1 1}] to [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:42:14.346785      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:14.347460      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    } {SUBMITTED } map[] 1 1} new status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:42:14.347513      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:14.347543      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:14.350320      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:42:14.354525      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:42:14.354569      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:20.202762      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:42:20.202863      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:20.202950      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:20.204075      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638477", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:42:20.206571      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:42:20.214030      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:42:20.214084      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:42:20.214103      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:20.214448      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:20.214484      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:20.217377      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:42:20.221451      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:42:20.221506      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:23.901974      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038901758-exec-1 updated in namespace spark.
I0319 23:42:23.902821      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038901758-exec-1 deleted in namespace spark.
I0319 23:42:28.738498      11 webhook.go:136] Serving admission request
I0319 23:42:28.748203      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038932554-exec-1 added in namespace spark.
I0319 23:42:28.748230      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:28.748280      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:28.748631      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638504", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038932554-exec-1 is pending
I0319 23:42:28.753176      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-1 updated in namespace spark.
I0319 23:42:28.753278      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:28.762850      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}]
I0319 23:42:28.766744      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-1 updated in namespace spark.
I0319 23:42:28.766803      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:28.770775      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}
I0319 23:42:28.770828      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:28.770863      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:28.771366      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638504", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038932554-exec-1 is pending
I0319 23:42:28.771752      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:28.773701      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}]
I0319 23:42:28.783402      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-2c586a8587364c70b9e3c4a94119d2f5 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}
I0319 23:42:28.783454      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:28.783483      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:28.786203      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}]
I0319 23:42:28.786362      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:28.814039      11 webhook.go:136] Serving admission request
I0319 23:42:28.823628      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553038932554-exec-2 added in namespace spark.
I0319 23:42:28.823690      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:28.823791      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:28.824311      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638536", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553038932554-exec-2 is pending
I0319 23:42:28.827855      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}]
I0319 23:42:28.828703      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-2 updated in namespace spark.
I0319 23:42:28.828762      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:28.835197      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:28.835436      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING] 1 1} new status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}
I0319 23:42:28.835556      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:28.835680      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:28.839202      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}]
I0319 23:42:28.839429      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:28.844185      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-2 updated in namespace spark.
I0319 23:42:28.844213      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:28.844239      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:28.847552      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}]
I0319 23:42:28.847701      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:34.013604      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-2 updated in namespace spark.
I0319 23:42:34.013646      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:34.013685      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:34.014021      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638542", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038932554-exec-2 is running
I0319 23:42:34.016737      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:RUNNING] 1 1}]
I0319 23:42:34.023385      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:PENDING] 1 1} new status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:RUNNING] 1 1}
I0319 23:42:34.023435      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038932554-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:42:34.023457      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:42:34.023477      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:34.023786      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:34.023829      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:34.026705      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:RUNNING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:RUNNING] 1 1}]
I0319 23:42:34.026833      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:34.083083      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-1 updated in namespace spark.
I0319 23:42:34.083112      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:42:34.083138      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:34.083347      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638570", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553038932554-exec-1 is running
I0319 23:42:34.085973      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:RUNNING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}]
I0319 23:42:34.092937      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:PENDING spark-pi-1553038932554-exec-2:RUNNING] 1 1} new status: {spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}
I0319 23:42:34.092987      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553038932554-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:42:34.093092      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:42:34.093110      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:34.093494      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:34.093533      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:34.096190      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}]
I0319 23:42:34.096339      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:42:43.912667      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:42:43.912733      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:42:43.921142      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}]
I0319 23:42:43.921329      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:13.912924      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:13.913105      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:13.921939      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}] to [{spark-application-1553038948495 spark-pi-1553038931258585742 2019-03-19 23:42:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30037    spark-pi-driver} {RUNNING } map[spark-pi-1553038932554-exec-1:RUNNING spark-pi-1553038932554-exec-2:RUNNING] 1 1}]
I0319 23:43:13.922092      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:25.820814      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:43:25.832658      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:43:25.846013      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:43:25.848714      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:43:25.849040      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:43:25.855291      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"78e43e18-4aa0-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638732", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationDeleted' SparkApplication spark-pi was deleted
I0319 23:43:25.861231      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-2 updated in namespace spark.
I0319 23:43:25.866329      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-1 updated in namespace spark.
I0319 23:43:26.640086      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-2 updated in namespace spark.
I0319 23:43:26.671552      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-1 updated in namespace spark.
I0319 23:43:27.658007      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-2 updated in namespace spark.
I0319 23:43:27.661176      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038932554-exec-2 deleted in namespace spark.
I0319 23:43:34.290318      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553038932554-exec-1 updated in namespace spark.
I0319 23:43:34.295015      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553038932554-exec-1 deleted in namespace spark.
I0319 23:43:40.355344      11 controller.go:172] SparkApplication spark/spark-pi was added, enqueueing it for submission
I0319 23:43:40.355401      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:40.355462      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:43:40.355555      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039020355487382 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039020355487382 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 10000]
I0319 23:43:40.355732      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638800", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationAdded' SparkApplication spark-pi was added, enqueuing it for submission
I0319 23:43:42.896483      11 webhook.go:136] Serving admission request
I0319 23:43:43.097904      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:43:43.108649      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:43:43.112966      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:43:43.124640      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:43:43.476312      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:43:43.476405      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:43:43.476598      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638800", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:43:43.488225      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0}] to [{ spark-pi-1553039020355487382 2019-03-19 23:43:43.476344533 +0000 UTC m=+420.621960059 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    } {SUBMITTED } map[] 1 1}]
I0319 23:43:43.496961      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0} new status: { spark-pi-1553039020355487382 2019-03-19 23:43:43.476344533 +0000 UTC m=+420.621960059 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    } {SUBMITTED } map[] 1 1}
I0319 23:43:43.497021      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:43.498809      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:43.498857      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:43.502073      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    } {SUBMITTED } map[] 1 1}] to [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:43:43.510001      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    } {SUBMITTED } map[] 1 1} new status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:43:43.510056      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:43.510606      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:43.510650      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:43.514412      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:43:43.518986      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:43:43.519039      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:43.913046      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:43.913096      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:43.916450      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:43:43.920563      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:43:43.920618      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:48.776960      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:43:48.776998      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:48.777031      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:48.777291      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638824", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:43:48.781803      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:43:48.790418      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:43:48.790472      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:43:48.790510      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:48.790863      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:48.790900      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:48.794086      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:43:48.798164      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:43:48.798215      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:53.079511      11 webhook.go:136] Serving admission request
I0319 23:43:53.090507      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039021724-exec-1 added in namespace spark.
I0319 23:43:53.090541      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:53.090577      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:53.090817      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638849", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039021724-exec-1 is pending
I0319 23:43:53.097166      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-1 updated in namespace spark.
I0319 23:43:53.097219      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:53.102764      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}]
I0319 23:43:53.110284      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}
I0319 23:43:53.110337      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:53.110368      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:53.110732      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638849", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039021724-exec-1 is pending
I0319 23:43:53.110818      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:53.111506      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-1 updated in namespace spark.
I0319 23:43:53.111526      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:53.113959      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}]
I0319 23:43:53.123036      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-01c0f9589f6e4ca285f926f8a90605f8 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}
I0319 23:43:53.123078      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:53.123097      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:53.125789      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}]
I0319 23:43:53.125923      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:53.147033      11 webhook.go:136] Serving admission request
I0319 23:43:53.157687      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039021724-exec-2 added in namespace spark.
I0319 23:43:53.157714      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:53.157803      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:53.158228      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638873", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039021724-exec-2 is pending
I0319 23:43:53.160758      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}]
I0319 23:43:53.164068      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-2 updated in namespace spark.
I0319 23:43:53.164095      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:53.168407      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1} new status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}
I0319 23:43:53.168507      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:53.168547      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:53.168715      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638873", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039021724-exec-2 is pending
I0319 23:43:53.168867      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:53.171953      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}]
I0319 23:43:53.177035      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-2 updated in namespace spark.
I0319 23:43:53.177065      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:53.182742      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING] 1 1} new status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}
I0319 23:43:53.182800      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:53.182835      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:53.185607      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}]
I0319 23:43:53.185740      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:58.029081      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-1 updated in namespace spark.
I0319 23:43:58.029120      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:58.029156      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:58.029471      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638880", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553039021724-exec-1 is running
I0319 23:43:58.032880      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:PENDING] 1 1}]
I0319 23:43:58.039472      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:PENDING spark-pi-1553039021724-exec-2:PENDING] 1 1} new status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:PENDING] 1 1}
I0319 23:43:58.039521      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553039021724-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:43:58.039541      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:43:58.039551      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:58.040546      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:58.040587      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:58.043617      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:PENDING] 1 1}]
I0319 23:43:58.043755      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:58.266727      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-2 updated in namespace spark.
I0319 23:43:58.266768      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:43:58.266804      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:58.267092      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638908", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553039021724-exec-2 is running
I0319 23:43:58.269925      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:PENDING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}]
I0319 23:43:58.277060      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:PENDING] 1 1} new status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}
I0319 23:43:58.277118      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553039021724-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:43:58.277150      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:43:58.277172      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:43:58.278127      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:43:58.278173      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:43:58.281036      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}]
I0319 23:43:58.281142      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:11.981566      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:11.981615      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:11.981774      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638955", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:44:11.982553      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:11.985120      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}] to [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {RUNNING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}]
I0319 23:44:11.985299      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:11.985338      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:11.985606      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:44:11.991400      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:11.991429      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:11.994621      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:44:11.994935      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:44:11.994995      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:12.020119      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {INVALIDATING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1}] to [{ spark-pi-1553039020355487382 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:44:12.033612      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-2 updated in namespace spark.
I0319 23:44:12.033678      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:12.034060      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-1 updated in namespace spark.
I0319 23:44:12.034088      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:12.075840      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039032638 spark-pi-1553039020355487382 2019-03-19 23:43:43 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {INVALIDATING } map[spark-pi-1553039021724-exec-1:RUNNING spark-pi-1553039021724-exec-2:RUNNING] 1 1} new status: { spark-pi-1553039020355487382 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:44:12.075912      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:12.075951      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:12.075955      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:12.089535      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:44:12.089630      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039052089565477 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039052089565477 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 20000]
I0319 23:44:12.089756      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638969", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:44:13.207012      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-1 updated in namespace spark.
I0319 23:44:13.207068      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:13.449162      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-2 updated in namespace spark.
I0319 23:44:13.449224      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:13.913284      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:14.571568      11 webhook.go:136] Serving admission request
I0319 23:44:14.572026      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:44:14.585392      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:44:14.590543      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:14.601422      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:14.937741      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:44:14.937829      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:44:14.938162      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2638969", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:44:14.951403      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039020355487382 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039052089565477 2019-03-19 23:44:14.937773888 +0000 UTC m=+452.083389413 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    } {SUBMITTED } map[] 1 1}]
I0319 23:44:14.966616      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039020355487382 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553039052089565477 2019-03-19 23:44:14.937773888 +0000 UTC m=+452.083389413 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    } {SUBMITTED } map[] 1 1}
I0319 23:44:14.966669      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:14.966705      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:14.968127      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:14.969330      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039020355487382 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039020355487382 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31119    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:44:14.969479      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:14.969521      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:14.972507      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    } {SUBMITTED } map[] 1 1}] to [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:44:14.979790      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:14.980102      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    } {SUBMITTED } map[] 1 1} new status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:44:14.980148      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:14.980178      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:14.983125      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:44:14.986945      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:44:14.986986      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:20.553403      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:20.553442      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:20.553473      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:20.553849      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639004", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:44:20.558931      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:44:20.567350      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:44:20.567406      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:44:20.567425      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:20.567634      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:20.567675      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:20.571831      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:44:20.577598      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:44:20.577651      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:22.607039      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-2 updated in namespace spark.
I0319 23:44:22.610132      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039021724-exec-2 deleted in namespace spark.
I0319 23:44:23.158090      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039021724-exec-1 updated in namespace spark.
I0319 23:44:23.161820      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039021724-exec-1 deleted in namespace spark.
I0319 23:44:29.739571      11 webhook.go:136] Serving admission request
I0319 23:44:29.749643      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039053375-exec-1 added in namespace spark.
I0319 23:44:29.749676      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:29.749713      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:29.750355      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639030", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039053375-exec-1 is pending
I0319 23:44:29.755748      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-1 updated in namespace spark.
I0319 23:44:29.755778      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:29.764639      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}]
I0319 23:44:29.774731      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-1 updated in namespace spark.
I0319 23:44:29.774765      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:29.778755      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}
I0319 23:44:29.778815      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:29.778847      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:29.779047      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639030", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039053375-exec-1 is pending
I0319 23:44:29.779792      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:29.782242      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}]
I0319 23:44:29.794012      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-512221dd4fd64694b40fd147ac43db72 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}
I0319 23:44:29.794057      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:29.794084      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:29.796887      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}]
I0319 23:44:29.796987      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:29.801779      11 webhook.go:136] Serving admission request
I0319 23:44:29.810787      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039053375-exec-2 added in namespace spark.
I0319 23:44:29.810816      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:29.810844      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:29.811140      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639067", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039053375-exec-2 is pending
I0319 23:44:29.814817      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}]
I0319 23:44:29.817132      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-2 updated in namespace spark.
I0319 23:44:29.817166      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:29.823162      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1} new status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}
I0319 23:44:29.823225      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:29.823276      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:29.823467      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639067", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039053375-exec-2 is pending
I0319 23:44:29.824573      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:29.827243      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}]
I0319 23:44:29.832132      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-2 updated in namespace spark.
I0319 23:44:29.832159      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:29.839219      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING] 1 1} new status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}
I0319 23:44:29.839298      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:29.839329      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:29.842042      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}]
I0319 23:44:29.842178      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:32.795553      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:32.795592      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:32.795771      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639087", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:44:32.796142      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:32.798652      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}] to [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {RUNNING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}]
I0319 23:44:32.798761      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:32.798794      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:32.798812      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:44:32.804738      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:32.804762      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:32.808014      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:44:32.808697      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:44:32.808718      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:32.824916      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-1 updated in namespace spark.
I0319 23:44:32.824987      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:32.832155      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-2 updated in namespace spark.
I0319 23:44:32.832253      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:32.842937      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {INVALIDATING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1}] to [{ spark-pi-1553039052089565477 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:44:32.846385      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-2 updated in namespace spark.
I0319 23:44:32.846420      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:32.849147      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-1 updated in namespace spark.
I0319 23:44:32.849300      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:32.852937      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039069467 spark-pi-1553039052089565477 2019-03-19 23:44:14 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {INVALIDATING } map[spark-pi-1553039053375-exec-1:PENDING spark-pi-1553039053375-exec-2:PENDING] 1 1} new status: { spark-pi-1553039052089565477 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:44:32.853012      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:32.853041      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:32.852982      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:32.862692      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:44:32.862778      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039072862715028 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039072862715028 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 30000]
I0319 23:44:32.863244      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639105", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:44:33.442916      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-2 updated in namespace spark.
I0319 23:44:33.442948      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:33.447207      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039053375-exec-2 deleted in namespace spark.
I0319 23:44:33.447240      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:34.822186      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039053375-exec-1 updated in namespace spark.
I0319 23:44:34.822233      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:34.826694      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039053375-exec-1 deleted in namespace spark.
I0319 23:44:34.827398      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:35.504286      11 webhook.go:136] Serving admission request
I0319 23:44:35.504713      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:44:35.516007      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:44:35.521982      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:35.538095      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:35.872644      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:44:35.872715      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:44:35.872848      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639105", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:44:35.885591      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039052089565477 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039072862715028 2019-03-19 23:44:35.872669619 +0000 UTC m=+473.018285032 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    } {SUBMITTED } map[] 1 1}]
I0319 23:44:35.896123      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039052089565477 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553039072862715028 2019-03-19 23:44:35.872669619 +0000 UTC m=+473.018285032 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    } {SUBMITTED } map[] 1 1}
I0319 23:44:35.896179      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:35.896351      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:35.897170      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:35.900667      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039052089565477 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039052089565477 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30983    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:44:35.900784      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:35.900827      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:35.903846      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    } {SUBMITTED } map[] 1 1}] to [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:44:35.913032      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    } {SUBMITTED } map[] 1 1} new status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:44:35.913087      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:35.913686      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:35.913741      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:35.917110      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:44:35.922153      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:44:35.922192      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:43.913465      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:43.913523      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:43.922745      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:44:43.928357      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:44:43.928401      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:44.462414      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:44:44.462450      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:44.462497      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:44.462755      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639142", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:44:44.465845      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:44:44.474277      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:44.474909      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:44:44.474959      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:44:44.476445      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:44.476490      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:44.479572      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:44:44.483640      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:44:44.483678      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:47.930332      11 webhook.go:136] Serving admission request
I0319 23:44:47.940876      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039074174-exec-1 added in namespace spark.
I0319 23:44:47.940902      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:47.940931      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:47.941187      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639176", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039074174-exec-1 is pending
I0319 23:44:47.945833      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}]
I0319 23:44:47.945970      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-1 updated in namespace spark.
I0319 23:44:47.946006      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:47.953549      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}
I0319 23:44:47.953596      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:47.953628      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:47.953789      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639176", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039074174-exec-1 is pending
I0319 23:44:47.954101      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:47.956410      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}]
I0319 23:44:47.961282      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-1 updated in namespace spark.
I0319 23:44:47.961337      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:47.966652      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-51ecbb4448934167810c516cb08dd2b1 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}
I0319 23:44:47.966697      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:47.966725      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:47.969608      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}]
I0319 23:44:47.969715      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:47.998659      11 webhook.go:136] Serving admission request
I0319 23:44:48.009477      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039074174-exec-2 added in namespace spark.
I0319 23:44:48.009505      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:48.009536      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:48.009997      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639191", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039074174-exec-2 is pending
I0319 23:44:48.013422      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}]
I0319 23:44:48.016093      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-2 updated in namespace spark.
I0319 23:44:48.016119      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:48.020760      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}
I0319 23:44:48.020805      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:48.020834      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:48.021056      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639191", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorPending' Executor spark-pi-1553039074174-exec-2 is pending
I0319 23:44:48.021361      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:48.024383      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}]
I0319 23:44:48.028420      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-2 updated in namespace spark.
I0319 23:44:48.028452      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:48.034789      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}
I0319 23:44:48.034833      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:48.034860      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:48.037560      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}]
I0319 23:44:48.037682      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:52.931002      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-2 updated in namespace spark.
I0319 23:44:52.931039      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:52.931078      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:52.931345      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639198", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553039074174-exec-2 is running
I0319 23:44:52.934459      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:44:52.941759      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:PENDING] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:RUNNING] 1 1}
I0319 23:44:52.941806      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553039074174-exec-2. OldState: PENDING NewState: RUNNING
I0319 23:44:52.941826      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:44:52.941841      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:52.942413      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:52.942454      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:52.945306      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:44:52.945422      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:53.066903      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-1 updated in namespace spark.
I0319 23:44:53.066945      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:44:53.066987      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:53.067325      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639224", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorRunning' Executor spark-pi-1553039074174-exec-1 is running
I0319 23:44:53.070407      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:44:53.077863      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:PENDING spark-pi-1553039074174-exec-2:RUNNING] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}
I0319 23:44:53.077912      11 sparkapp_metrics.go:204] Exporting Metrics for Executor spark-pi-1553039074174-exec-1. OldState: PENDING NewState: RUNNING
I0319 23:44:53.077927      11 metrics.go:111] Incrementing spark_app_executor_running_count with labels map[app_type:Unknown]
I0319 23:44:53.077936      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:44:53.078699      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:44:53.078741      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:44:53.081452      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:44:53.081561      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:45:13.913660      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:45:13.913723      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:45:13.924206      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:45:13.924388      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:45:43.913863      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:45:43.913932      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:45:43.924916      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:45:43.925101      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:46:13.914026      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:46:13.914089      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:46:13.925213      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:46:13.925433      11 controller.go:252] Ending processing key: "spark/spark-pi"
W0319 23:46:25.969185      11 reflector.go:341] github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions/factory.go:119: watch of *v1beta1.ScheduledSparkApplication ended with: The resourceVersion for the provided watch is too old.
I0319 23:46:43.914214      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:46:43.914330      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:46:43.924276      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:46:43.924433      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:47:13.914337      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:47:13.914437      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:47:13.924164      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:47:13.924388      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:47:43.914511      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:47:43.914584      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:47:43.923673      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:47:43.923857      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:03.626367      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-1 updated in namespace spark.
I0319 23:48:03.626402      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:03.626446      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:03.634609      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-2 updated in namespace spark.
I0319 23:48:03.634659      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:03.635483      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:48:03.635603      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:03.635647      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:03.638434      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}]
I0319 23:48:03.638531      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:04.281607      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-2 updated in namespace spark.
I0319 23:48:04.281640      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:04.281680      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:04.281989      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639228", FieldPath:""}): type: 'Normal' reason: 'SparkExecutorCompleted' Executor spark-pi-1553039074174-exec-2 completed
I0319 23:48:04.285210      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:04.293947      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:RUNNING] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}
I0319 23:48:04.293991      11 sparkapp_metrics.go:210] Exporting Metrics for Executor spark-pi-1553039074174-exec-2. OldState: RUNNING NewState: COMPLETED
I0319 23:48:04.294008      11 metrics.go:124] Decrementing spark_app_executor_running_count with labels map[app_type:Unknown] metricVal to 21
I0319 23:48:04.294025      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:04.294213      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:48:04.294288      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:04.296996      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:04.297126      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:04.538282      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:48:04.538308      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:04.538327      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:04.538586      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639875", FieldPath:""}): type: 'Normal' reason: 'SparkDriverCompleted' Driver spark-pi-driver completed
I0319 23:48:04.543205      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04.538408911 +0000 UTC m=+681.684024919 {spark-pi-ui-svc 32700    spark-pi-driver} {SUCCEEDING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:04.551817      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:48:04.552595      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {RUNNING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04.538408911 +0000 UTC m=+681.684024919 {spark-pi-ui-svc 32700    spark-pi-driver} {SUCCEEDING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}
I0319 23:48:04.552734      11 metrics.go:124] Decrementing spark_app_running_count with labels map[app_type:Unknown] metricVal to 11
I0319 23:48:04.552773      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:04.552804      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:04.552846      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUCCEEDING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:04.553305      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2639881", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationCompleted' SparkApplication spark-pi completed
I0319 23:48:04.560053      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {SUCCEEDING } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1} new status: {spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}
I0319 23:48:04.560130      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:04.561708      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:48:04.561809      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:04.561880      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:04.562056      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:04.566190      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-1 updated in namespace spark.
I0319 23:48:04.566236      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:04.566316      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:04.566376      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:04.566538      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:05.300067      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-2 updated in namespace spark.
I0319 23:48:05.300095      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:05.300127      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:05.300167      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:05.300357      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:05.303392      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039074174-exec-2 deleted in namespace spark.
I0319 23:48:05.303416      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:05.303441      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:05.303461      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:05.303577      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:13.888677      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039074174-exec-1 updated in namespace spark.
I0319 23:48:13.888708      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:13.888747      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:13.888792      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:13.889012      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:13.893507      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039074174-exec-1 deleted in namespace spark.
I0319 23:48:13.893531      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:48:13.893550      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:13.893573      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:13.893700      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:13.914703      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:48:13.914735      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:13.914760      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:13.914851      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:48:43.914912      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:48:43.914968      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:48:43.915003      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:48:43.915132      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:49:13.915182      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:49:13.915338      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:49:13.915411      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:49:13.915608      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:49:43.915359      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:49:43.915423      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:49:43.915463      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:49:43.915613      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:50:13.915556      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:50:13.915608      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:50:13.915638      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:50:13.915780      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:50:43.915733      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:50:43.915801      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:50:43.915840      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:50:43.916021      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:51:13.915926      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:51:13.915983      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:51:13.916015      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:51:13.916159      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:51:43.916228      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:51:43.916353      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:51:43.916424      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:51:43.916636      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:52:13.916445      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:52:13.916556      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:52:13.916590      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:52:13.916740      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:52:43.916655      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:52:43.916721      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:52:43.916759      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:52:43.916928      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:53:13.916849      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:53:13.916908      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:53:13.916955      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:53:13.917105      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:53:43.917047      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:53:43.917100      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:53:43.917132      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:53:43.917294      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:54:13.917246      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:54:13.917335      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:54:13.917385      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:54:13.917552      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:54:43.917434      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:54:43.917495      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:54:43.917535      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:54:43.917730      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:55:13.917643      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:55:13.917700      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:55:13.917738      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:55:13.917929      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:55:43.917833      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:55:43.917885      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:55:43.917925      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}] to [{spark-application-1553039087681 spark-pi-1553039072862715028 2019-03-19 23:44:35 +0000 UTC 2019-03-19 23:48:04 +0000 UTC {spark-pi-ui-svc 32700    spark-pi-driver} {COMPLETED } map[spark-pi-1553039074174-exec-1:RUNNING spark-pi-1553039074174-exec-2:COMPLETED] 1 1}]
I0319 23:55:43.918068      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:55:50.036137      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:55:50.048563      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:55:50.051899      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:55:50.052043      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:55:50.055304      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"d2ce5238-4aa0-11e9-91f6-0aee7538e092", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641347", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationDeleted' SparkApplication spark-pi was deleted
I0319 23:55:53.612856      11 controller.go:172] SparkApplication spark/spark-pi was added, enqueueing it for submission
I0319 23:55:53.612918      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:55:53.613016      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:55:53.613109      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039753613039257 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039753613039257 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 10000]
I0319 23:55:53.613561      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641376", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationAdded' SparkApplication spark-pi was added, enqueuing it for submission
I0319 23:55:56.117395      11 webhook.go:136] Serving admission request
I0319 23:55:56.117870      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:55:56.127488      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:55:56.131853      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:55:56.145494      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:55:56.443302      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:55:56.443379      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:55:56.443594      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641376", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:55:56.455686      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0}] to [{ spark-pi-1553039753613039257 2019-03-19 23:55:56.443328387 +0000 UTC m=+1153.588943867 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    } {SUBMITTED } map[] 1 1}]
I0319 23:55:56.468114      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {  0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC { 0    } { } map[] 0 0} new status: { spark-pi-1553039753613039257 2019-03-19 23:55:56.443328387 +0000 UTC m=+1153.588943867 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    } {SUBMITTED } map[] 1 1}
I0319 23:55:56.468174      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:55:56.469084      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:55:56.469119      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:55:56.472305      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    } {SUBMITTED } map[] 1 1}] to [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:55:56.479058      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    } {SUBMITTED } map[] 1 1} new status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:55:56.479142      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:55:56.481345      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:55:56.481388      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:55:56.484510      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:55:56.488218      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:55:56.488290      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:02.588972      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:02.589069      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:02.589134      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:02.589550      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641402", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:56:02.598828      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:02.608297      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:02.608350      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:56:02.608370      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:02.610988      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:02.611107      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:02.615157      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:02.620519      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:02.620576      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:04.826125      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:04.826786      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:04.826677      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641438", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:56:04.826827      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:04.827056      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:56:04.836398      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:04.836547      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:04.840802      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:56:04.846413      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:56:04.846448      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:04.873935      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {INVALIDATING } map[] 1 1}] to [{ spark-pi-1553039753613039257 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:56:04.882853      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {INVALIDATING } map[] 1 1} new status: { spark-pi-1553039753613039257 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:56:04.882898      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:04.882932      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:04.882973      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:56:04.883204      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:04.885993      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:56:04.888487      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {INVALIDATING } map[] 1 1}] to [{ spark-pi-1553039753613039257 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:56:04.898658      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-17f182db6e0f4b4ab9bc88940f4d577f spark-pi-1553039753613039257 2019-03-19 23:55:56 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {INVALIDATING } map[] 1 1} new status: { spark-pi-1553039753613039257 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:56:04.898697      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:04.898718      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:04.903376      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:56:04.903483      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039764903418773 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039764903418773 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 20000]
I0319 23:56:04.903579      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641452", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:56:07.435942      11 webhook.go:136] Serving admission request
I0319 23:56:07.436417      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:56:07.445166      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:56:07.450589      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:07.463658      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:07.775844      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:56:07.775917      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:56:07.776142      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641452", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:56:07.788916      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039753613039257 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039764903418773 2019-03-19 23:56:07.775870842 +0000 UTC m=+1164.921486287 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    } {SUBMITTED } map[] 1 1}]
I0319 23:56:07.800322      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:07.800942      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039753613039257 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31310    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553039764903418773 2019-03-19 23:56:07.775870842 +0000 UTC m=+1164.921486287 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    } {SUBMITTED } map[] 1 1}
I0319 23:56:07.801052      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:07.801118      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:07.804585      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    } {SUBMITTED } map[] 1 1}] to [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:56:07.811796      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    } {SUBMITTED } map[] 1 1} new status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:56:07.811845      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:07.812109      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:07.812146      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:07.815133      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:56:07.819420      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:56:07.819475      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:13.444100      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:13.444132      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:13.444175      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:13.444495      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641481", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:56:13.454912      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:13.463555      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:13.463617      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:56:13.463647      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:13.466045      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:13.466091      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:13.469005      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:13.473467      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:13.473516      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:13.918158      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:13.918275      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:13.921734      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:13.926293      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:13.926336      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:15.626200      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:15.626240      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:15.626251      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641516", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:56:15.626440      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:15.629385      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:15.641610      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:15.641650      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:15.641678      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:15.641707      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:56:15.642231      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:15.648563      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:15.648596      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:15.652017      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:56:15.652217      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:56:15.652270      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:15.678216      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {INVALIDATING } map[] 1 1}] to [{ spark-pi-1553039764903418773 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:56:15.697553      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:15.697817      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-b3a8aebb8c3f484183a63a3b35ca21d7 spark-pi-1553039764903418773 2019-03-19 23:56:07 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {INVALIDATING } map[] 1 1} new status: { spark-pi-1553039764903418773 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:56:15.697861      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:15.697889      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:15.702277      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:56:15.702366      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641532", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:56:15.702414      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039775702310039 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039775702310039 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 30000]
I0319 23:56:18.046707      11 webhook.go:136] Serving admission request
I0319 23:56:18.047095      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:56:18.056756      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:56:18.061595      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:18.072033      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:18.427771      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:56:18.428158      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:56:18.428566      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641532", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:56:18.442346      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039764903418773 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039775702310039 2019-03-19 23:56:18.428012783 +0000 UTC m=+1175.573628590 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    } {SUBMITTED } map[] 1 1}]
I0319 23:56:18.450709      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039764903418773 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 30558    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553039775702310039 2019-03-19 23:56:18.428012783 +0000 UTC m=+1175.573628590 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    } {SUBMITTED } map[] 1 1}
I0319 23:56:18.450762      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:18.452542      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:18.452601      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:18.456819      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    } {SUBMITTED } map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:56:18.464418      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:18.464544      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    } {SUBMITTED } map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:56:18.464584      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:18.464614      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:18.467570      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1}]
I0319 23:56:18.471342      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1}
I0319 23:56:18.471399      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:24.359391      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:24.359421      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:24.359453      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:24.359702      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641556", FieldPath:""}): type: 'Normal' reason: 'SparkDriverRunning' Driver spark-pi-driver is running
I0319 23:56:24.370878      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:24.378083      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {SUBMITTED } map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:24.378130      11 metrics.go:111] Incrementing spark_app_running_count with labels map[app_type:Unknown]
I0319 23:56:24.378143      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:24.378889      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:24.378941      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:24.382761      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:24.387178      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:24.387225      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:26.490088      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:26.490132      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:26.490315      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641592", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSpecUpdateProcessed' Successfully processed spec update for SparkApplication spark-pi
I0319 23:56:26.490681      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:26.493295      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}]
I0319 23:56:26.507154      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}
I0319 23:56:26.507202      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:26.507234      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:26.507283      11 controller.go:699] Deleting pod with name spark-pi-driver in namespace spark
I0319 23:56:26.507495      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:26.513557      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:26.513585      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:26.517153      11 controller.go:708] Deleting Spark UI Service spark-pi-ui-svc in namespace spark
I0319 23:56:26.517241      11 spark_pod_eventhandler.go:77] Pod spark-pi-driver deleted in namespace spark.
I0319 23:56:26.517291      11 spark_pod_eventhandler.go:95] Enqueuing SparkApplication spark/spark-pi for app update processing.
I0319 23:56:26.542436      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {INVALIDATING } map[] 1 1}] to [{ spark-pi-1553039775702310039 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {PENDING_RERUN } map[] 0 0}]
I0319 23:56:26.556801      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {INVALIDATING } map[] 1 1} new status: { spark-pi-1553039775702310039 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {PENDING_RERUN } map[] 0 0}
I0319 23:56:26.556841      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:26.556871      11 controller.go:245] Starting processing key: "spark/spark-pi"
W0319 23:56:26.556953      11 controller.go:334] driver not found for SparkApplication: spark/spark-pi
I0319 23:56:26.556975      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26.556966857 +0000 UTC m=+1183.702582450 {spark-pi-ui-svc 31660    spark-pi-driver} {FAILING Driver Pod not found} map[] 1 1}]
I0319 23:56:26.558776      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:26.569674      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {RUNNING } map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26.556966857 +0000 UTC m=+1183.702582450 {spark-pi-ui-svc 31660    spark-pi-driver} {FAILING Driver Pod not found} map[] 1 1}
I0319 23:56:26.569751      11 metrics.go:124] Decrementing spark_app_running_count with labels map[app_type:Unknown] metricVal to 13
I0319 23:56:26.569777      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:26.569804      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:26.570292      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:26.574455      11 monitoring_config.go:54] Overriding the default Prometheus configuration with config file /etc/metrics/conf/prometheus.yaml in the Spark image.
I0319 23:56:26.574540      11 submission.go:63] spark-submit arguments: [/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master k8s://https://172.30.0.1:443 --deploy-mode cluster --conf spark.kubernetes.namespace=spark --conf spark.app.name=spark-pi --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.kubernetes.container.image=lightbend/spark:2.1.0-OpenShift-2.4.0-rh --conf spark.kubernetes.container.image.pullPolicy=Always --conf spark.kubernetes.submission.waitAppCompletion=false --conf spark.metrics.namespace=spark.spark-pi --conf spark.metrics.conf=/etc/metrics/conf/metrics.properties --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.driver.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039786574481132 --conf spark.kubernetes.driver.pod.name=spark-pi-driver --conf spark.driver.cores=1.000000 --conf spark.driver.memory=1g --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-sa --conf spark.kubernetes.driver.label.version=2.4.0 --conf spark.kubernetes.driver.annotation.prometheus.io/scrape=true --conf spark.kubernetes.driver.annotation.prometheus.io/port=9100 --conf spark.kubernetes.driver.annotation.prometheus.io/path=/metrics --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.driver.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.driver.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/app-name=spark-pi --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/launched-by-spark-operator=true --conf spark.kubernetes.executor.label.sparkoperator.k8s.io/submission-id=spark-pi-1553039786574481132 --conf spark.executor.instances=2 --conf spark.executor.cores=2 --conf spark.executor.memory=1g --conf spark.kubernetes.executor.label.version=2.4.0 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/port=9100 --conf spark.kubernetes.executor.annotation.xxxxxxx.lightbend.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/scrape=true --conf spark.kubernetes.executor.annotation.prometheus.io/port=9100 --conf spark.kubernetes.executor.annotation.prometheus.io/path=/metrics --conf spark.executor.extraJavaOptions=-javaagent:/prometheus/jmx_prometheus_javaagent-0.3.1.jar=9100:/etc/metrics/conf/prometheus.yaml local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar 10000]
I0319 23:56:26.574688      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641607", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationPendingRerun' SparkApplication spark-pi is pending rerun
I0319 23:56:29.049609      11 webhook.go:136] Serving admission request
I0319 23:56:29.050040      11 webhook.go:290] Pod spark-pi-driver in namespace spark is subject to mutation
I0319 23:56:29.059750      11 spark_pod_eventhandler.go:47] Pod spark-pi-driver added in namespace spark.
I0319 23:56:29.066879      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:29.078389      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:29.396891      11 controller.go:596] SparkApplication spark/spark-pi has been submitted
I0319 23:56:29.396969      11 sparkui.go:135] Creating a service spark-pi-ui-svc for the Spark UI for application spark-pi
I0319 23:56:29.397181      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641607", FieldPath:""}): type: 'Normal' reason: 'SparkApplicationSubmitted' SparkApplication spark-pi was submitted successfully
I0319 23:56:29.411047      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{ spark-pi-1553039775702310039 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {PENDING_RERUN } map[] 0 0}] to [{ spark-pi-1553039786574481132 2019-03-19 23:56:29.396916509 +0000 UTC m=+1186.542531986 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31007    } {SUBMITTED } map[] 1 1}]
I0319 23:56:29.454104      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: { spark-pi-1553039775702310039 0001-01-01 00:00:00 +0000 UTC 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {PENDING_RERUN } map[] 0 0} new status: { spark-pi-1553039786574481132 2019-03-19 23:56:29.396916509 +0000 UTC m=+1186.542531986 0001-01-01 00:00:00 +0000 UTC {spark-pi-ui-svc 31007    } {SUBMITTED } map[] 1 1}
I0319 23:56:29.454153      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:29.454190      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:29.454232      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILING Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:56:29.454690      11 event.go:221] Event(v1.ObjectReference{Kind:"SparkApplication", Namespace:"spark", Name:"spark-pi", UID:"87dd2218-4aa2-11e9-b674-0288a0a0c3ec", APIVersion:"sparkoperator.k8s.io/v1beta1", ResourceVersion:"2641608", FieldPath:""}): type: 'Warning' reason: 'SparkApplicationFailed' SparkApplication spark-pi failed: Driver Pod not found
I0319 23:56:29.455003      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:29.494486      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:29.496302      11 sparkapp_metrics.go:144] Exporting metrics for spark-pi; old status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILING Driver Pod not found} map[] 1 1} new status: {spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}
I0319 23:56:29.496341      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:29.496365      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:29.496395      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:56:29.496496      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:35.606810      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:56:43.918302      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:56:43.918371      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:56:43.918411      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:56:43.918571      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:56:44.603132      11 webhook.go:136] Serving admission request
I0319 23:56:44.615936      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039787910-exec-1 added in namespace spark.
I0319 23:56:44.621574      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-1 updated in namespace spark.
I0319 23:56:44.633318      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-1 updated in namespace spark.
I0319 23:56:44.686233      11 webhook.go:136] Serving admission request
I0319 23:56:44.695511      11 spark_pod_eventhandler.go:47] Pod spark-pi-1553039787910-exec-2 added in namespace spark.
I0319 23:56:44.700761      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-2 updated in namespace spark.
I0319 23:56:44.712486      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-2 updated in namespace spark.
I0319 23:56:49.672126      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-2 updated in namespace spark.
I0319 23:56:49.842555      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-1 updated in namespace spark.
I0319 23:57:13.918500      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:57:13.918564      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:57:13.918603      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:57:13.918740      11 controller.go:252] Ending processing key: "spark/spark-pi"
W0319 23:57:31.988801      11 reflector.go:341] github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/client/informers/externalversions/factory.go:119: watch of *v1beta1.ScheduledSparkApplication ended with: The resourceVersion for the provided watch is too old.
I0319 23:57:43.918694      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:57:43.918763      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:57:43.918799      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:57:43.918923      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:58:13.918893      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:58:13.918955      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:58:13.918997      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:58:13.919143      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:58:42.806502      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-1 updated in namespace spark.
I0319 23:58:42.814497      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-2 updated in namespace spark.
I0319 23:58:43.919119      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:58:43.919175      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:58:43.919212      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:58:43.919363      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:58:44.041556      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-2 updated in namespace spark.
I0319 23:58:44.197379      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-1 updated in namespace spark.
I0319 23:58:44.230292      11 spark_pod_eventhandler.go:58] Pod spark-pi-driver updated in namespace spark.
I0319 23:58:45.058674      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-2 updated in namespace spark.
I0319 23:58:45.061846      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039787910-exec-2 deleted in namespace spark.
I0319 23:58:45.214465      11 spark_pod_eventhandler.go:58] Pod spark-pi-1553039787910-exec-1 updated in namespace spark.
I0319 23:58:45.218167      11 spark_pod_eventhandler.go:77] Pod spark-pi-1553039787910-exec-1 deleted in namespace spark.
I0319 23:59:13.919383      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:59:13.919465      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:59:13.919497      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:59:13.919630      11 controller.go:252] Ending processing key: "spark/spark-pi"
I0319 23:59:43.919692      11 controller.go:205] SparkApplication spark/spark-pi was updated, enqueueing it
I0319 23:59:43.919777      11 controller.go:245] Starting processing key: "spark/spark-pi"
I0319 23:59:43.919815      11 controller.go:522] Trying to update SparkApplication spark/spark-pi, from: [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}] to [{spark-f3e575fefe4444939ca9bf66a69bb464 spark-pi-1553039775702310039 2019-03-19 23:56:18 +0000 UTC 2019-03-19 23:56:26 +0000 UTC {spark-pi-ui-svc 31660    spark-pi-driver} {FAILED Driver Pod not found} map[] 1 1}]
I0319 23:59:43.919990      11 controller.go:252] Ending processing key: "spark/spark-pi"