gistfile1.txt

E0324 19:04:43.304806       1 reflector.go:134] github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:282: Failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:default:promtail" cannot list pods at the cluster scope
level=debug ts=2019-03-24T19:04:49.135627749Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.32.0.4\", __meta_kubernetes_pod_container_name=\"fluentd-gcp\"}"
level=debug ts=2019-03-24T19:04:49.138892921Z caller=filetargetmanager.go:144 msg="ignoring target, wrong host" labels="{__address__=\"10.32.0.4\", __host__=\"gke-primary-production-default-pool-a94f2735-037l\", __meta_kubernetes_namespace=\"kube-system\", __meta_kubernetes_pod_annotation_scheduler_alpha_kubernetes_io_critical_pod=\"\", __meta_kubernetes_pod_container_name=\"fluentd-gcp\", __meta_kubernetes_pod_controller_kind=\"DaemonSet\", __meta_kubernetes_pod_controller_name=\"fluentd-gcp-v3.2.0\", __meta_kubernetes_pod_host_ip=\"10.138.0.13\", __meta_kubernetes_pod_ip=\"10.32.0.4\", __meta_kubernetes_pod_label_controller_revision_hash=\"169971793\", __meta_kubernetes_pod_label_k8s_app=\"fluentd-gcp\", __meta_kubernetes_pod_label_kubernetes_io_cluster_service=\"true\", __meta_kubernetes_pod_label_pod_template_generation=\"2\", __meta_kubernetes_pod_label_version=\"v3.2.0\", __meta_kubernetes_pod_name=\"fluentd-gcp-v3.2.0-7mmk9\", __meta_kubernetes_pod_node_name=\"gke-primary-production-default-pool-a94f2735-037l\", __meta_kubernetes_pod_phase=\"Running\", __meta_kubernetes_pod_ready=\"true\", __meta_kubernetes_pod_uid=\"e0d3b6c6-4d4e-11e9-8a35-42010a8a00e6\", __path__=\"/var/log/pods/e0d3b6c6-4d4e-11e9-8a35-42010a8a00e6/fluentd-gcp/*.log\", __service__=\"fluentd-gcp-v3.2.0\", container_name=\"fluentd-gcp\", controller_revision_hash=\"169971793\", instance=\"fluentd-gcp-v3.2.0-7mmk9\", job=\"kube-system/fluentd-gcp-v3.2.0\", k8s_app=\"fluentd-gcp\", kubernetes_io_cluster_service=\"true\", namespace=\"kube-system\", pod_template_generation=\"2\", version=\"v3.2.0\"}" hostname=gke-primary-production-default-pool-a94f2735-n9t2
level=debug ts=2019-03-24T19:04:49.138985892Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.32.0.4\", __meta_kubernetes_pod_container_name=\"prometheus-to-sd-exporter\"}"
level=debug ts=2019-03-24T19:04:49.13908278Z caller=filetargetmanager.go:144 msg="ignoring target, wrong host" labels="{__address__=\"10.32.0.4\", __host__=\"gke-primary-production-default-pool-a94f2735-037l\", __meta_kubernetes_namespace=\"kube-system\", __meta_kubernetes_pod_annotation_scheduler_alpha_kubernetes_io_critical_pod=\"\", __meta_kubernetes_pod_container_name=\"prometheus-to-sd-exporter\", __meta_kubernetes_pod_controller_kind=\"DaemonSet\", __meta_kubernetes_pod_controller_name=\"fluentd-gcp-v3.2.0\", __meta_kubernetes_pod_host_ip=\"10.138.0.13\", __meta_kubernetes_pod_ip=\"10.32.0.4\", __meta_kubernetes_pod_label_controller_revision_hash=\"169971793\", __meta_kubernetes_pod_label_k8s_app=\"fluentd-gcp\", __meta_kubernetes_pod_label_kubernetes_io_cluster_service=\"true\", __meta_kubernetes_pod_label_pod_template_generation=\"2\", __meta_kubernetes_pod_label_version=\"v3.2.0\", __meta_kubernetes_pod_name=\"fluentd-gcp-v3.2.0-7mmk9\", __meta_kubernetes_pod_node_name=\"gke-primary-production-default-pool-a94f2735-037l\", __meta_kubernetes_pod_phase=\"Running\", __meta_kubernetes_pod_ready=\"true\", __meta_kubernetes_pod_uid=\"e0d3b6c6-4d4e-11e9-8a35-42010a8a00e6\", __path__=\"/var/log/pods/e0d3b6c6-4d4e-11e9-8a35-42010a8a00e6/prometheus-to-sd-exporter/*.log\", __service__=\"fluentd-gcp-v3.2.0\", container_name=\"prometheus-to-sd-exporter\", controller_revision_hash=\"169971793\", instance=\"fluentd-gcp-v3.2.0-7mmk9\", job=\"kube-system/fluentd-gcp-v3.2.0\", k8s_app=\"fluentd-gcp\", kubernetes_io_cluster_service=\"true\", namespace=\"kube-system\", pod_template_generation=\"2\", version=\"v3.2.0\"}" hostname=gke-primary-production-default-pool-a94f2735-n9t2
level=debug ts=2019-03-24T19:04:49.139097129Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.32.2.14:9200\", __meta_kubernetes_pod_container_name=\"elasticsearch\", __meta_kubernetes_pod_container_port_name=\"\", __meta_kubernetes_pod_container_port_number=\"9200\", __meta_kubernetes_pod_container_port_protocol=\"TCP\"}"
level=debug ts=2019-03-24T19:04:49.139193514Z caller=filetargetmanager.go:144 msg="ignoring target, wrong host" labels="{__address__=\"10.32.2.14:9200\", __host__=\"gke-primary-production-default-pool-a94f2735-1zcg\", __meta_kubernetes_namespace=\"default\", __meta_kubernetes_pod_annotation_kubernetes_io_limit_ranger=\"LimitRanger plugin set: cpu request for container elasticsearch; cpu request for init container configure-sysctl\", __meta_kubernetes_pod_container_name=\"elasticsearch\", __meta_kubernetes_pod_container_port_name=\"\", __meta_kubernetes_pod_container_port_number=\"9200\", __meta_kubernetes_pod_container_port_protocol=\"TCP\", __meta_kubernetes_pod_controller_kind=\"StatefulSet\", __meta_kubernetes_pod_controller_name=\"elasticsearch\", __meta_kubernetes_pod_host_ip=\"10.138.0.14\", __meta_kubernetes_pod_ip=\"10.32.2.14\", __meta_kubernetes_pod_label_controller_revision_hash=\"elasticsearch-647895fbd9\", __meta_kubernetes_pod_label_group=\"elasticsearch\", __meta_kubernetes_pod_label_role=\"database\", __meta_kubernetes_pod_label_statefulset_kubernetes_io_pod_name=\"elasticsearch-1\", __meta_kubernetes_pod_name=\"elasticsearch-1\", __meta_kubernetes_pod_node_name=\"gke-primary-production-default-pool-a94f2735-1zcg\", __meta_kubernetes_pod_phase=\"Running\", __meta_kubernetes_pod_ready=\"true\", __meta_kubernetes_pod_uid=\"fdad5f3c-4d4e-11e9-8a35-42010a8a00e6\", __path__=\"/var/log/pods/fdad5f3c-4d4e-11e9-8a35-42010a8a00e6/elasticsearch/*.log\", __service__=\"elasticsearch\", container_name=\"elasticsearch\", controller_revision_hash=\"elasticsearch-647895fbd9\", group=\"elasticsearch\", instance=\"elasticsearch-1\", job=\"default/elasticsearch\", namespace=\"default\", role=\"database\", statefulset_kubernetes_io_pod_name=\"elasticsearch-1\"}" hostname=gke-primary-production-default-pool-a94f2735-n9t2
level=debug ts=2019-03-24T19:04:49.139207305Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.32.2.14:9300\", __meta_kubernetes_pod_container_name=\"elasticsearch\", __meta_kubernetes_pod_container_port_name=\"\", __meta_kubernetes_pod_container_port_number=\"9300\", __meta_kubernetes_pod_container_port_protocol=\"TCP\"}"
level=debug ts=2019-03-24T19:04:49.139304069Z caller=filetargetmanager.go:144 msg="ignoring target, wrong host" labels="{__address__=\"10.32.2.14:9300\", __host__=\"gke-primary-production-default-pool-a94f2735-1zcg\", __meta_kubernetes_namespace=\"default\", __meta_kubernetes_pod_annotation_kubernetes_io_limit_ranger=\"LimitRanger plugin set: cpu request for container elasticsearch; cpu request for init container configure-sysctl\", __meta_kubernetes_pod_container_name=\"elasticsearch\", __meta_kubernetes_pod_container_port_name=\"\", __meta_kubernetes_pod_container_port_number=\"9300\", __meta_kubernetes_pod_container_port_protocol=\"TCP\", __meta_kubernetes_pod_controller_kind=\"StatefulSet\", __meta_kubernetes_pod_controller_name=\"elasticsearch\", __meta_kubernetes_pod_host_ip=\"10.138.0.14\", __meta_kubernetes_pod_ip=\"10.32.2.14\", __meta_kubernetes_pod_label_controller_revision_hash=\"elasticsearch-647895fbd9\", __meta_kubernetes_pod_label_group=\"elasticsearch\", __meta_kubernetes_pod_label_role=\"database\", __meta_kubernetes_pod_label_statefulset_kubernetes_io_pod_name=\"elasticsearch-1\", __meta_kubernetes_pod_name=\"elasticsearch-1\", __meta_kubernetes_pod_node_name=\"gke-primary-production-default-pool-a94f2735-1zcg\", __meta_kubernetes_pod_phase=\"Running\", __meta_kubernetes_pod_ready=\"true\", __meta_kubernetes_pod_uid=\"fdad5f3c-4d4e-11e9-8a35-42010a8a00e6\", __path__=\"/var/log/pods/fdad5f3c-4d4e-11e9-8a35-42010a8a00e6/elasticsearch/*.log\", __service__=\"elasticsearch\", container_name=\"elasticsearch\", controller_revision_hash=\"elasticsearch-647895fbd9\", group=\"elasticsearch\", instance=\"elasticsearch-1\", job=\"default/elasticsearch\", namespace=\"default\", role=\"database\", statefulset_kubernetes_io_pod_name=\"elasticsearch-1\"}" hostname=gke-primary-production-default-pool-a94f2735-n9t2
level=debug ts=2019-03-24T19:04:49.13931669Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.138.0.13\", __meta_kubernetes_pod_container_name=\"metricbeat\"}"
level=debug ts=2019-03-24T19:04:49.139432749Z caller=filetargetmanager.go:144 msg="ignoring target, wrong host" labels="{__address__=\"10.138.0.13\", __host__=\"gke-primary-production-default-pool-a94f2735-037l\", __meta_kubernetes_namespace=\"kube-system\", __meta_kubernetes_pod_container_name=\"metricbeat\", __meta_kubernetes_pod_controller_kind=\"DaemonSet\", __meta_kubernetes_pod_controller_name=\"metricbeat\", __meta_kubernetes_pod_host_ip=\"10.138.0.13\", __meta_kubernetes_pod_ip=\"10.138.0.13\", __meta_kubernetes_pod_label_controller_revision_hash=\"3975653690\", __meta_kubernetes_pod_label_group=\"metricbeat\", __meta_kubernetes_pod_label_pod_template_generation=\"1\", __meta_kubernetes_pod_label_role=\"monitoring\", __meta_kubernetes_pod_name=\"metricbeat-v7kbs\", __meta_kubernetes_pod_node_name=\"gke-primary-production-default-pool-a94f2735-037l\", __meta_kubernetes_pod_phase=\"Running\", __meta_kubernetes_pod_ready=\"true\", __meta_kubernetes_pod_uid=\"f3c8284b-4d5b-11e9-8a35-42010a8a00e6\", __path__=\"/var/log/pods/f3c8284b-4d5b-11e9-8a35-42010a8a00e6/metricbeat/*.log\", __service__=\"metricbeat\", container_name=\"metricbeat\", controller_revision_hash=\"3975653690\", group=\"metricbeat\", instance=\"metricbeat-v7kbs\", job=\"kube-system/metricbeat\", namespace=\"kube-system\", pod_template_generation=\"1\", role=\"monitoring\"}" hostname=gke-primary-production-default-pool-a94f2735-n9t2
level=debug ts=2019-03-24T19:04:49.139448597Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.138.0.16\", __meta_kubernetes_pod_container_name=\"metricbeat\"}"
level=info ts=2019-03-24T19:04:49.139506684Z caller=filetargetmanager.go:170 msg="Adding target" key="{container_name=\"metricbeat\", controller_revision_hash=\"3975653690\", group=\"metricbeat\", instance=\"metricbeat-zqngf\", job=\"kube-system/metricbeat\", namespace=\"kube-system\", pod_template_generation=\"1\", role=\"monitoring\"}"
level=debug ts=2019-03-24T19:04:49.13964116Z caller=filetarget.go:212 msg="watching new directory" directory=/var/log/pods/f3c0f335-4d5b-11e9-8a35-42010a8a00e6/metricbeat
level=debug ts=2019-03-24T19:04:49.139676325Z caller=filetarget.go:246 msg="tailing new file" filename=/var/log/pods/f3c0f335-4d5b-11e9-8a35-42010a8a00e6/metricbeat/0.log
level=debug ts=2019-03-24T19:04:49.13971438Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.32.2.6\", __meta_kubernetes_pod_container_name=\"autoscaler\"}"
level=debug ts=2019-03-24T19:04:49.139733202Z caller=filetargetmanager.go:137 msg="dropping target, no labels"
level=debug ts=2019-03-24T19:04:49.139743508Z caller=filetargetmanager.go:130 msg="new target" labels="{__address__=\"10.32.2.5\", __meta_kubernetes_pod_container_name=\"fluentd-gcp-scaler\"}"
level=debug ts=2019-03-24T19:04:49.139762001Z caller=filetargetmanager.go:137 msg="dropping target, no labels"