dionysius/servicedns_steps.sh

## servicedns_steps.sh
$ kubectl config get-contexts
CURRENT   NAME       CLUSTER         AUTHINFO                         NAMESPACE
*         cluster0   mgmt-test      kubernetes-admin@mgmt-test
          cluster1   member-test    kubernetes-admin@member-test
          cluster2   member-test2   kubernetes-admin@member-test2

$ for c in cluster0 cluster1 cluster2; do echo "###$c:###"; helm --kube-context "$c" list; done
###cluster0:###
NAME                            REVISION        UPDATED                         STATUS          CHART                                           APP VERSION     NAMESPACE
kubefed                         1               Mon Aug 12 18:42:35 2019        DEPLOYED        kubefed-0.1.0-rc5                                               kube-federation-system
metallb                         1               Tue Aug 13 14:57:00 2019        DEPLOYED        metallb-0.10.0                                  0.8.1           metallb-system
###cluster1:###
NAME                            REVISION        UPDATED                         STATUS          CHART                                           APP VERSION     NAMESPACE
metallb                         1               Wed Aug 14 18:33:08 2019        DEPLOYED        metallb-0.11.1                                  0.8.1           metallb-system
###cluster2:###
NAME                            REVISION        UPDATED                         STATUS          CHART                                           APP VERSION     NAMESPACE
metallb                         1               Tue Aug 13 14:59:21 2019        DEPLOYED        metallb-0.10.0                                  0.8.1           metallb-system

# one exception to the kubefed-controller: image is edited to use :canary and runs with argument --v 6

$ cat example/sample1/federatedservice.yaml | sed 's/NodePort/LoadBalancer/' | kubectl create -f -
federatedservice.types.kubefed.io/test-service created

$ kubectl create -f example/sample1/federateddeployment.yaml
federateddeployment.types.kubefed.io/test-deployment created

$ cat <<EOF | kubectl create -f -
> apiVersion: multiclusterdns.kubefed.io/v1alpha1
> kind: Domain
> metadata:
>   # Corresponds to <federation> in the resource records.
>   name: test-domain
>   # The namespace running kubefed-controller-manager.
>   namespace: kube-federation-system
> # The domain/subdomain that is setup in your externl-dns provider.
> domain: example.com
> ---
> apiVersion: multiclusterdns.kubefed.io/v1alpha1
> kind: ServiceDNSRecord
> metadata:
>   # The name of the sample service.
>   name: test-service
>   # The namespace of the sample deployment/service.
>   namespace: test-namespace
> spec:
>   # The name of the corresponding Domain.
>   domainRef: test-domain
>   recordTTL: 300
> EOF
domain.multiclusterdns.kubefed.io/test-domain created
servicednsrecord.multiclusterdns.kubefed.io/test-service created

$ for c in cluster0 cluster1 cluster2; do echo "###$c:###"; kubectl --context "$c" -n test-namespace get pod; kubectl --context "$c" -n test-namespace get service; kubectl --context "$c" -n test-namespace get endpoints; done
###cluster0:###
No resources found.
No resources found.
No resources found.
###cluster1:###
NAME                               READY   STATUS    RESTARTS   AGE
test-deployment-554b9c67f9-4w96m   1/1     Running   0          9m37s
test-deployment-554b9c67f9-fwmz4   1/1     Running   0          9m37s
test-deployment-554b9c67f9-v6gv7   1/1     Running   0          9m37s
NAME           TYPE           CLUSTER-IP       EXTERNAL-IP     PORT(S)        AGE
test-service   LoadBalancer   10.103.186.172   192.168.1.240   80:30987/TCP   9m36s
NAME           ENDPOINTS                                   AGE
test-service   172.18.0.2:80,172.18.0.3:80,172.18.0.5:80   9m36s
###cluster2:###
NAME                               READY   STATUS    RESTARTS   AGE
test-deployment-5bf5c5558d-8pk2v   1/1     Running   0          9m39s
test-deployment-5bf5c5558d-b857q   1/1     Running   0          9m39s
test-deployment-5bf5c5558d-gwdk5   1/1     Running   0          9m39s
test-deployment-5bf5c5558d-nd8bb   1/1     Running   0          9m39s
test-deployment-5bf5c5558d-prmtz   1/1     Running   0          9m39s
NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
test-service   LoadBalancer   10.109.130.66   192.168.1.240   80:30599/TCP   9m38s
NAME           ENDPOINTS                                               AGE
test-service   172.18.0.2:80,172.18.0.3:80,172.18.0.4:80 + 2 more...   9m39s

$ kubectl -n test-namespace get servicednsrecords.multiclusterdns.kubefed.io test-service -o yaml
apiVersion: multiclusterdns.kubefed.io/v1alpha1
kind: ServiceDNSRecord
metadata:
  creationTimestamp: "2019-08-16T12:21:08Z"
  generation: 1
  name: test-service
  namespace: test-namespace
  resourceVersion: "3795286"
  selfLink: /apis/multiclusterdns.kubefed.io/v1alpha1/namespaces/test-namespace/servicednsrecords/test-service
  uid: 22a43706-1585-4773-8c9e-1fa0537f4e39
spec:
  domainRef: test-domain
  recordTTL: 300

$ kubectl -n test-namespace get dnsendpoints.multiclusterdns.kubefed.io service-test-service -o yaml
apiVersion: multiclusterdns.kubefed.io/v1alpha1
kind: DNSEndpoint
metadata:
  creationTimestamp: "2019-08-16T12:21:09Z"
  generation: 1
  name: service-test-service
  namespace: test-namespace
  resourceVersion: "3795306"
  selfLink: /apis/multiclusterdns.kubefed.io/v1alpha1/namespaces/test-namespace/dnsendpoints/service-test-service
  uid: 30fce8c1-6e49-46bd-af91-a9c688d95b34
spec: {}
status:
  observedGeneration: 1

$ kubectl -n kube-federation-system logs kubefed-controller-manager-57b8cdbc95-bwqvk | grep -i ServiceDNS
I0816 10:28:33.699589       1 controller.go:88] Starting ServiceDNS controller
I0816 10:28:33.700424       1 reflector.go:131] Starting reflector *v1alpha1.ServiceDNSRecord (0s) from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197
I0816 10:28:33.700468       1 reflector.go:169] Listing and watching *v1alpha1.ServiceDNSRecord from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197
I0816 10:28:33.701320       1 reflector.go:131] Starting reflector *v1alpha1.Domain (0s) from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198
I0816 10:28:33.701518       1 reflector.go:169] Listing and watching *v1alpha1.Domain from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198
[...]
I0816 12:21:09.284149       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:21:09.382027       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 1.022733ms)
I0816 12:21:14.382746       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:21:14.382963       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 113.384µs)
I0816 12:21:24.384805       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:21:24.385527       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 364.987µs)
I0816 12:21:25.783772       1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198: Watch close - *v1alpha1.Domain total 1 items received
I0816 12:21:44.388427       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:21:44.389190       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 135.106µs)
I0816 12:22:12.888869       1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197: Watch close - *v1alpha1.ServiceDNSRecord total 1 items received
I0816 12:22:12.891692       1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795286&timeoutSeconds=419&watch=true 200 OK in 2 milliseconds
I0816 12:22:24.391370       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:22:24.391937       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 98.504µs)
I0816 12:23:24.481876       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:24.485620       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 89.032µs)
I0816 12:23:50.893329       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:50.893437       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 23.663µs)
I0816 12:23:50.982938       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:50.986474       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 39.185µs)
I0816 12:23:50.986555       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:50.986572       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 5.64µs)
I0816 12:23:52.384396       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:52.385006       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 554.126µs)
I0816 12:23:52.388354       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:52.388386       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 10.731µs)
I0816 12:23:52.388416       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:52.388429       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 5.751µs)
I0816 12:23:54.483725       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:54.483905       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 112.86µs)
I0816 12:23:56.388503       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:23:56.396264       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 28.921µs)
I0816 12:24:00.992034       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:00.992816       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 137.442µs)
I0816 12:24:03.492931       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:03.492991       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 24.197µs)
I0816 12:24:05.482254       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:05.482310       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 17.581µs)
I0816 12:24:06.382459       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:06.382572       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 55.034µs)
I0816 12:24:08.482815       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:08.483010       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 124.955µs)
I0816 12:24:13.486742       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:13.486957       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 25.535µs)
I0816 12:24:15.188839       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:15.188920       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 21.241µs)
I0816 12:24:17.181660       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:17.183059       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 26.982µs)
I0816 12:24:19.386614       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:19.387210       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 465.883µs)
I0816 12:24:24.389161       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:24.391268       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 153.806µs)
I0816 12:24:34.485356       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:34.585200       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 101.854µs)
I0816 12:24:54.688034       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:24:54.688208       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 56.606µs)
I0816 12:25:22.882018       1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/dnsendpoint/controller.go:116: Watch close - *v1alpha1.ServiceDNSRecord total 2 items received
I0816 12:25:22.981017       1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795642&timeoutSeconds=546&watch=true 200 OK in 5 milliseconds
I0816 12:25:34.689077       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:25:34.689424       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 202.732µs)
I0816 12:26:34.689807       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:26:34.690176       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 253.286µs)
I0816 12:27:34.690619       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:27:34.691122       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 103.171µs)
I0816 12:28:30.881626       1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198: Watch close - *v1alpha1.Domain total 0 items received
I0816 12:28:34.787223       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:28:34.787791       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 141.961µs)
I0816 12:29:11.982205       1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197: Watch close - *v1alpha1.ServiceDNSRecord total 1 items received
I0816 12:29:11.989920       1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795642&timeoutSeconds=482&watch=true 200 OK in 7 milliseconds
I0816 12:29:34.791815       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:29:34.791964       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 41.642µs)
I0816 12:30:34.795260       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:30:34.795391       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 30.391µs)
I0816 12:31:35.181003       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:31:35.181773       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 211.724µs)
I0816 12:32:35.182485       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:32:35.183079       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 137.039µs)
I0816 12:33:35.201888       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:33:35.202020       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 35.337µs)
I0816 12:34:28.986082       1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/dnsendpoint/controller.go:116: Watch close - *v1alpha1.ServiceDNSRecord total 0 items received
I0816 12:34:29.087667       1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795642&timeoutSeconds=498&watch=true 200 OK in 6 milliseconds
I0816 12:34:35.289250       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:34:35.289739       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 131.638µs)
I0816 12:35:35.291174       1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
I0816 12:35:35.291748       1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 85.179µs)

$ kubectl -n kube-federation-system logs kubefed-controller-manager-57b8cdbc95-bwqvk | grep -E '(W|E)0816'
W0816 10:28:08.281020       1 client_config.go:549] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
[...]
W0816 12:00:19.882164       1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 220879 (221435)
W0816 12:13:53.481158       1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 2644808 (2645146)
W0816 12:22:10.295712       1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 222778 (223170)
W0816 12:45:06.688642       1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 224747 (225056)
W0816 12:46:36.789968       1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 2647403 (2648047)
	$ kubectl config get-contexts
	CURRENT NAME CLUSTER AUTHINFO NAMESPACE
	* cluster0 mgmt-test kubernetes-admin@mgmt-test
	cluster1 member-test kubernetes-admin@member-test
	cluster2 member-test2 kubernetes-admin@member-test2

	$ for c in cluster0 cluster1 cluster2; do echo "###$c:###"; helm --kube-context "$c" list; done
	###cluster0:###
	NAME REVISION UPDATED STATUS CHART APP VERSION NAMESPACE
	kubefed 1 Mon Aug 12 18:42:35 2019 DEPLOYED kubefed-0.1.0-rc5 kube-federation-system
	metallb 1 Tue Aug 13 14:57:00 2019 DEPLOYED metallb-0.10.0 0.8.1 metallb-system
	###cluster1:###
	NAME REVISION UPDATED STATUS CHART APP VERSION NAMESPACE
	metallb 1 Wed Aug 14 18:33:08 2019 DEPLOYED metallb-0.11.1 0.8.1 metallb-system
	###cluster2:###
	NAME REVISION UPDATED STATUS CHART APP VERSION NAMESPACE
	metallb 1 Tue Aug 13 14:59:21 2019 DEPLOYED metallb-0.10.0 0.8.1 metallb-system

	# one exception to the kubefed-controller: image is edited to use :canary and runs with argument --v 6

	$ cat example/sample1/federatedservice.yaml \| sed 's/NodePort/LoadBalancer/' \| kubectl create -f -
	federatedservice.types.kubefed.io/test-service created

	$ kubectl create -f example/sample1/federateddeployment.yaml
	federateddeployment.types.kubefed.io/test-deployment created

	$ cat <<EOF \| kubectl create -f -
	> apiVersion: multiclusterdns.kubefed.io/v1alpha1
	> kind: Domain
	> metadata:
	> # Corresponds to <federation> in the resource records.
	> name: test-domain
	> # The namespace running kubefed-controller-manager.
	> namespace: kube-federation-system
	> # The domain/subdomain that is setup in your externl-dns provider.
	> domain: example.com
	> ---
	> apiVersion: multiclusterdns.kubefed.io/v1alpha1
	> kind: ServiceDNSRecord
	> metadata:
	> # The name of the sample service.
	> name: test-service
	> # The namespace of the sample deployment/service.
	> namespace: test-namespace
	> spec:
	> # The name of the corresponding Domain.
	> domainRef: test-domain
	> recordTTL: 300
	> EOF
	domain.multiclusterdns.kubefed.io/test-domain created
	servicednsrecord.multiclusterdns.kubefed.io/test-service created

	$ for c in cluster0 cluster1 cluster2; do echo "###$c:###"; kubectl --context "$c" -n test-namespace get pod; kubectl --context "$c" -n test-namespace get service; kubectl --context "$c" -n test-namespace get endpoints; done
	###cluster0:###
	No resources found.
	No resources found.
	No resources found.
	###cluster1:###
	NAME READY STATUS RESTARTS AGE
	test-deployment-554b9c67f9-4w96m 1/1 Running 0 9m37s
	test-deployment-554b9c67f9-fwmz4 1/1 Running 0 9m37s
	test-deployment-554b9c67f9-v6gv7 1/1 Running 0 9m37s
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	test-service LoadBalancer 10.103.186.172 192.168.1.240 80:30987/TCP 9m36s
	NAME ENDPOINTS AGE
	test-service 172.18.0.2:80,172.18.0.3:80,172.18.0.5:80 9m36s
	###cluster2:###
	NAME READY STATUS RESTARTS AGE
	test-deployment-5bf5c5558d-8pk2v 1/1 Running 0 9m39s
	test-deployment-5bf5c5558d-b857q 1/1 Running 0 9m39s
	test-deployment-5bf5c5558d-gwdk5 1/1 Running 0 9m39s
	test-deployment-5bf5c5558d-nd8bb 1/1 Running 0 9m39s
	test-deployment-5bf5c5558d-prmtz 1/1 Running 0 9m39s
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	test-service LoadBalancer 10.109.130.66 192.168.1.240 80:30599/TCP 9m38s
	NAME ENDPOINTS AGE
	test-service 172.18.0.2:80,172.18.0.3:80,172.18.0.4:80 + 2 more... 9m39s

	$ kubectl -n test-namespace get servicednsrecords.multiclusterdns.kubefed.io test-service -o yaml
	apiVersion: multiclusterdns.kubefed.io/v1alpha1
	kind: ServiceDNSRecord
	metadata:
	creationTimestamp: "2019-08-16T12:21:08Z"
	generation: 1
	name: test-service
	namespace: test-namespace
	resourceVersion: "3795286"
	selfLink: /apis/multiclusterdns.kubefed.io/v1alpha1/namespaces/test-namespace/servicednsrecords/test-service
	uid: 22a43706-1585-4773-8c9e-1fa0537f4e39
	spec:
	domainRef: test-domain
	recordTTL: 300

	$ kubectl -n test-namespace get dnsendpoints.multiclusterdns.kubefed.io service-test-service -o yaml
	apiVersion: multiclusterdns.kubefed.io/v1alpha1
	kind: DNSEndpoint
	metadata:
	creationTimestamp: "2019-08-16T12:21:09Z"
	generation: 1
	name: service-test-service
	namespace: test-namespace
	resourceVersion: "3795306"
	selfLink: /apis/multiclusterdns.kubefed.io/v1alpha1/namespaces/test-namespace/dnsendpoints/service-test-service
	uid: 30fce8c1-6e49-46bd-af91-a9c688d95b34
	spec: {}
	status:
	observedGeneration: 1

	$ kubectl -n kube-federation-system logs kubefed-controller-manager-57b8cdbc95-bwqvk \| grep -i ServiceDNS
	I0816 10:28:33.699589 1 controller.go:88] Starting ServiceDNS controller
	I0816 10:28:33.700424 1 reflector.go:131] Starting reflector *v1alpha1.ServiceDNSRecord (0s) from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197
	I0816 10:28:33.700468 1 reflector.go:169] Listing and watching *v1alpha1.ServiceDNSRecord from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197
	I0816 10:28:33.701320 1 reflector.go:131] Starting reflector *v1alpha1.Domain (0s) from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198
	I0816 10:28:33.701518 1 reflector.go:169] Listing and watching *v1alpha1.Domain from sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198
	[...]
	I0816 12:21:09.284149 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:21:09.382027 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 1.022733ms)
	I0816 12:21:14.382746 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:21:14.382963 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 113.384µs)
	I0816 12:21:24.384805 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:21:24.385527 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 364.987µs)
	I0816 12:21:25.783772 1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198: Watch close - *v1alpha1.Domain total 1 items received
	I0816 12:21:44.388427 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:21:44.389190 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 135.106µs)
	I0816 12:22:12.888869 1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197: Watch close - *v1alpha1.ServiceDNSRecord total 1 items received
	I0816 12:22:12.891692 1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795286&timeoutSeconds=419&watch=true 200 OK in 2 milliseconds
	I0816 12:22:24.391370 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:22:24.391937 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 98.504µs)
	I0816 12:23:24.481876 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:24.485620 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 89.032µs)
	I0816 12:23:50.893329 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:50.893437 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 23.663µs)
	I0816 12:23:50.982938 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:50.986474 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 39.185µs)
	I0816 12:23:50.986555 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:50.986572 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 5.64µs)
	I0816 12:23:52.384396 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:52.385006 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 554.126µs)
	I0816 12:23:52.388354 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:52.388386 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 10.731µs)
	I0816 12:23:52.388416 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:52.388429 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 5.751µs)
	I0816 12:23:54.483725 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:54.483905 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 112.86µs)
	I0816 12:23:56.388503 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:23:56.396264 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 28.921µs)
	I0816 12:24:00.992034 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:00.992816 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 137.442µs)
	I0816 12:24:03.492931 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:03.492991 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 24.197µs)
	I0816 12:24:05.482254 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:05.482310 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 17.581µs)
	I0816 12:24:06.382459 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:06.382572 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 55.034µs)
	I0816 12:24:08.482815 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:08.483010 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 124.955µs)
	I0816 12:24:13.486742 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:13.486957 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 25.535µs)
	I0816 12:24:15.188839 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:15.188920 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 21.241µs)
	I0816 12:24:17.181660 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:17.183059 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 26.982µs)
	I0816 12:24:19.386614 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:19.387210 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 465.883µs)
	I0816 12:24:24.389161 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:24.391268 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 153.806µs)
	I0816 12:24:34.485356 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:34.585200 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 101.854µs)
	I0816 12:24:54.688034 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:24:54.688208 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 56.606µs)
	I0816 12:25:22.882018 1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/dnsendpoint/controller.go:116: Watch close - *v1alpha1.ServiceDNSRecord total 2 items received
	I0816 12:25:22.981017 1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795642&timeoutSeconds=546&watch=true 200 OK in 5 milliseconds
	I0816 12:25:34.689077 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:25:34.689424 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 202.732µs)
	I0816 12:26:34.689807 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:26:34.690176 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 253.286µs)
	I0816 12:27:34.690619 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:27:34.691122 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 103.171µs)
	I0816 12:28:30.881626 1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:198: Watch close - *v1alpha1.Domain total 0 items received
	I0816 12:28:34.787223 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:28:34.787791 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 141.961µs)
	I0816 12:29:11.982205 1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/servicedns/controller.go:197: Watch close - *v1alpha1.ServiceDNSRecord total 1 items received
	I0816 12:29:11.989920 1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795642&timeoutSeconds=482&watch=true 200 OK in 7 milliseconds
	I0816 12:29:34.791815 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:29:34.791964 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 41.642µs)
	I0816 12:30:34.795260 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:30:34.795391 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 30.391µs)
	I0816 12:31:35.181003 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:31:35.181773 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 211.724µs)
	I0816 12:32:35.182485 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:32:35.183079 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 137.039µs)
	I0816 12:33:35.201888 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:33:35.202020 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 35.337µs)
	I0816 12:34:28.986082 1 reflector.go:357] sigs.k8s.io/kubefed/pkg/controller/dnsendpoint/controller.go:116: Watch close - *v1alpha1.ServiceDNSRecord total 0 items received
	I0816 12:34:29.087667 1 round_trippers.go:438] GET https://10.96.0.1:443/apis/multiclusterdns.kubefed.io/v1alpha1/servicednsrecords?resourceVersion=3795642&timeoutSeconds=498&watch=true 200 OK in 6 milliseconds
	I0816 12:34:35.289250 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:34:35.289739 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 131.638µs)
	I0816 12:35:35.291174 1 controller.go:266] Starting to reconcile ServiceDNS resource: test-namespace/test-service
	I0816 12:35:35.291748 1 controller.go:269] Finished reconciling ServiceDNS resource test-namespace/test-service (duration: 85.179µs)

	$ kubectl -n kube-federation-system logs kubefed-controller-manager-57b8cdbc95-bwqvk \| grep -E '(W\|E)0816'
	W0816 10:28:08.281020 1 client_config.go:549] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.
	[...]
	W0816 12:00:19.882164 1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 220879 (221435)
	W0816 12:13:53.481158 1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 2644808 (2645146)
	W0816 12:22:10.295712 1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 222778 (223170)
	W0816 12:45:06.688642 1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 224747 (225056)
	W0816 12:46:36.789968 1 reflector.go:270] sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go:437: watch of <nil> ended with: too old resource version: 2647403 (2648047)