distributor.log

level=info ts=2023-08-21T14:37:25.783057863Z caller=main.go:221 msg="initialising OpenTracing tracer"
level=info ts=2023-08-21T14:37:25.783684155Z caller=main.go:108 msg="Starting Tempo" version="(version=release-v2.2-a48e5bb, branch=release-v2.2, revision=a48e5bb46)"
level=info ts=2023-08-21T14:37:25.784695573Z caller=server.go:354 http=[::]:3200 grpc=[::]:9095 msg="server listening on addresses"
level=info ts=2023-08-21T14:37:25.785204163Z caller=memberlist_client.go:437 msg="Using memberlist cluster label and node name" cluster_label= node=tempo-simplest-distributor-79b945c757-8dl69-70e666ad
ts=2023-08-21T14:37:25Z level=info msg="OTel Shim Logger Initialized" component=tempo
level=info ts=2023-08-21T14:37:25.785917246Z caller=memberlist_client.go:543 msg="memberlist fast-join starting" nodes_found=1 to_join=2
level=info ts=2023-08-21T14:37:25.78615942Z caller=module_service.go:82 msg=initialising module=internal-server
level=info ts=2023-08-21T14:37:25.786933804Z caller=module_service.go:82 msg=initialising module=server
level=info ts=2023-08-21T14:37:25.78725188Z caller=module_service.go:82 msg=initialising module=overrides
level=info ts=2023-08-21T14:37:25.787319611Z caller=module_service.go:82 msg=initialising module=memberlist-kv
level=info ts=2023-08-21T14:37:25.787349442Z caller=module_service.go:82 msg=initialising module=metrics-generator-ring
level=info ts=2023-08-21T14:37:25.787344132Z caller=module_service.go:82 msg=initialising module=ring
level=info ts=2023-08-21T14:37:25.789122585Z caller=memberlist_client.go:563 msg="memberlist fast-join finished" joined_nodes=4 elapsed_time=3.206969ms
level=info ts=2023-08-21T14:37:25.789155715Z caller=memberlist_client.go:576 msg="joining memberlist cluster" join_members=tempo-simplest-gossip-ring
level=info ts=2023-08-21T14:37:25.789177466Z caller=ring.go:262 msg="ring doesn't exist in KV store yet"
level=info ts=2023-08-21T14:37:25.789230767Z caller=module_service.go:82 msg=initialising module=distributor
ts=2023-08-21T14:37:25Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks" component=tempo documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2023-08-21T14:37:25Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks" component=tempo documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2023-08-21T14:37:25Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks" component=tempo documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2023-08-21T14:37:25Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks" component=tempo documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2023-08-21T14:37:25Z level=info msg="Starting GRPC server" component=tempo endpoint=0.0.0.0:4317
ts=2023-08-21T14:37:25Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks" component=tempo documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2023-08-21T14:37:25Z level=info msg="Starting HTTP server" component=tempo endpoint=0.0.0.0:4318
level=info ts=2023-08-21T14:37:25.789513332Z caller=app.go:196 msg="Tempo started"
level=info ts=2023-08-21T14:37:25.79050288Z caller=memberlist_client.go:595 msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=1.346925ms
ts=2023-08-21T14:37:28.988062102Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.62:7946"
ts=2023-08-21T14:37:30.989152644Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.63:7946"
ts=2023-08-21T14:37:30.989394848Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.66:7946"
ts=2023-08-21T14:37:34.791563592Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.66:7946"
ts=2023-08-21T14:37:36.792717876Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.67:7946"
ts=2023-08-21T14:37:36.792756056Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.63:7946"
ts=2023-08-21T14:37:36.792929389Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.62:7946"
ts=2023-08-21T14:37:38.283124668Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.67:7946"
ts=2023-08-21T14:37:40.284361103Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.66:7946"
ts=2023-08-21T14:37:40.284387734Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.63:7946"
ts=2023-08-21T14:37:40.284527106Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.62:7946"
ts=2023-08-21T14:37:40.786592657Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.63:7946"
ts=2023-08-21T14:37:42.787802481Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.66:7946"
ts=2023-08-21T14:37:42.787858622Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.67:7946"
ts=2023-08-21T14:37:42.788127747Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.62:7946"
ts=2023-08-21T14:37:43.988992798Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.62:7946"
ts=2023-08-21T14:37:45.786943983Z caller=memberlist_logger.go:74 level=info msg="Suspect tempo-simplest-distributor-79b945c757-8dl69-54667208 has failed, no acks received"
ts=2023-08-21T14:37:45.990737922Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.66:7946"
ts=2023-08-21T14:37:45.990784862Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.67:7946"
ts=2023-08-21T14:37:45.990804833Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.63:7946"
ts=2023-08-21T14:37:49.791464269Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.66:7946"
ts=2023-08-21T14:37:51.79247065Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.63:7946"
ts=2023-08-21T14:37:51.792698224Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.62:7946"
ts=2023-08-21T14:37:51.792868527Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'tempo-simplest-distributor-79b945c757-8dl69-54667208' from=10.244.0.67:7946"
ts=2023-08-21T14:37:53.989730281Z caller=memberlist_logger.go:74 level=info msg="Marking tempo-simplest-distributor-79b945c757-8dl69-54667208 as failed, suspect timeout reached (2 peer confirmations)"