Skip to content

Instantly share code, notes, and snippets.

@SIMULATAN

SIMULATAN/mimir.log Secret

Created March 21, 2023 18:10
Show Gist options
  • Save SIMULATAN/c529220bc236e6f8e04332777d215c3e to your computer and use it in GitHub Desktop.
Save SIMULATAN/c529220bc236e6f8e04332777d215c3e to your computer and use it in GitHub Desktop.
Download ZIP
Raw
mimir.log
ts=2023-03-21T18:06:15.614611419Z caller=main.go:210 level=info msg="Starting application" version="(version=2.7.0, branch=HEAD, revision=4af9b3a)"
ts=2023-03-21T18:06:15.616707186Z caller=server.go:322 level=info http=[::]:9009 grpc=[::]:9095 msg="server listening on addresses"
ts=2023-03-21T18:06:15.632402472Z caller=ingester.go:336 level=info msg="TSDB idle compaction timeout set" timeout=1h1m12.497021164s
ts=2023-03-21T18:06:15.634603198Z caller=inmemory.go:173 level=info msg="created in-memory index cache" maxItemSizeBytes=134217728 maxSizeBytes=1073741824 maxItems=maxInt
ts=2023-03-21T18:06:15.634669399Z caller=cache.go:105 level=info msg="created chunks cache"
ts=2023-03-21T18:06:15.635616001Z caller=memberlist_client.go:437 level=info msg="Using memberlist cluster label and node name" cluster_label= node=be7c11639b37-0cdfe93d
ts=2023-03-21T18:06:15.639607573Z caller=log.go:194 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
ts=2023-03-21T18:06:15.640467096Z caller=modules.go:434 level=info msg="The querier worker has not been configured with either the query-frontend or query-scheduler address. Because Mimir is running in monolithic mode, it's attempting an automatic worker configuration. If queries are unresponsive, consider explicitly configuring the query-frontend or query-scheduler address for querier worker." address=127.0.0.1:9095
ts=2023-03-21T18:06:15.640530776Z caller=worker.go:130 level=info msg="Starting querier worker connected to query-frontend" frontend=127.0.0.1:9095
ts=2023-03-21T18:06:15.641389498Z caller=mapper.go:52 level=info msg="cleaning up mapped rules directory" path=./data-ruler/
ts=2023-03-21T18:06:15.642499342Z caller=api.go:143 level=debug msg="api: registering route" methods=DELETE path=/prometheus/config/v1/rules/{namespace}/{groupName} auth=true gzip=true
ts=2023-03-21T18:06:15.642573822Z caller=api.go:143 level=debug msg="api: registering route" methods=DELETE path=/prometheus/config/v1/rules/{namespace} auth=true gzip=true
ts=2023-03-21T18:06:15.643144224Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=server
ts=2023-03-21T18:06:15.643301504Z caller=module_service.go:82 level=info msg=initialising module=sanity-check
ts=2023-03-21T18:06:15.643356064Z caller=sanity_check.go:32 level=info msg="Checking directories read/write access"
ts=2023-03-21T18:06:15.643502265Z caller=module_service.go:82 level=info msg=initialising module=usage-stats
ts=2023-03-21T18:06:15.643671465Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-queryable waiting_for=activity-tracker
ts=2023-03-21T18:06:15.643709225Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ring waiting_for=usage-stats
ts=2023-03-21T18:06:15.643747145Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=query-frontend waiting_for=sanity-check
ts=2023-03-21T18:06:15.643799026Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=distributor-service waiting_for=sanity-check
ts=2023-03-21T18:06:15.643829466Z caller=module_service.go:82 level=info msg=initialising module=active-groups-cleanup-service
ts=2023-03-21T18:06:15.643871106Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=memberlist-kv waiting_for=activity-tracker
ts=2023-03-21T18:06:15.643908746Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-gateway waiting_for=usage-stats
ts=2023-03-21T18:06:15.644156587Z caller=module_service.go:82 level=info msg=initialising module=activity-tracker
ts=2023-03-21T18:06:15.644216307Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=compactor waiting_for=sanity-check
ts=2023-03-21T18:06:15.644266027Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=server waiting_for=activity-tracker
ts=2023-03-21T18:06:15.644304987Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=server waiting_for=sanity-check
ts=2023-03-21T18:06:15.644334747Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ingester-service waiting_for=activity-tracker
ts=2023-03-21T18:06:15.644552068Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ingester-service waiting_for=memberlist-kv
ts=2023-03-21T18:06:15.644390787Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=memberlist-kv waiting_for=sanity-check
ts=2023-03-21T18:06:15.644400227Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-queryable waiting_for=memberlist-kv
ts=2023-03-21T18:06:15.644515268Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=store-queryable
ts=2023-03-21T18:06:15.644447067Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-gateway waiting_for=activity-tracker
ts=2023-03-21T18:06:15.661424958Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-gateway waiting_for=memberlist-kv
ts=2023-03-21T18:06:15.660913396Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ring waiting_for=activity-tracker
ts=2023-03-21T18:06:15.661470958Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
ts=2023-03-21T18:06:15.661844239Z caller=sanity_check.go:37 level=info msg="Directories read/write access successfully checked"
ts=2023-03-21T18:06:15.661900839Z caller=sanity_check.go:39 level=info msg="Checking object storage config"
ts=2023-03-21T18:06:15.663952685Z caller=tcp_transport.go:393 level=debug component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=192.168.192.5 advertisePort=7946
ts=2023-03-21T18:06:15.681956778Z caller=tcp_transport.go:393 level=debug component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=192.168.192.5 advertisePort=7946
ts=2023-03-21T18:06:20.289796425Z caller=reporter.go:133 level=info msg="usage stats reporter initialized" cluster_id=844711e1-86f8-479b-a32a-a0196c79596d
ts=2023-03-21T18:06:24.324335779Z caller=sanity_check.go:44 level=info msg="Object storage config successfully checked"
ts=2023-03-21T18:06:24.324388219Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=memberlist-kv waiting_for=server
ts=2023-03-21T18:06:24.324402139Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=server waiting_for=usage-stats
ts=2023-03-21T18:06:24.324409619Z caller=module_service.go:82 level=info msg=initialising module=server
ts=2023-03-21T18:06:24.324440419Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=query-frontend waiting_for=server
ts=2023-03-21T18:06:24.324465019Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=compactor waiting_for=server
ts=2023-03-21T18:06:24.324476859Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=distributor-service waiting_for=server
ts=2023-03-21T18:06:24.32450034Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=distributor-service waiting_for=usage-stats
ts=2023-03-21T18:06:24.3245095Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=distributor-service waiting_for=activity-tracker
ts=2023-03-21T18:06:24.32451678Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=distributor-service waiting_for=memberlist-kv
ts=2023-03-21T18:06:24.3245233Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=compactor waiting_for=usage-stats
ts=2023-03-21T18:06:24.32453122Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=compactor waiting_for=activity-tracker
ts=2023-03-21T18:06:24.32453662Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
ts=2023-03-21T18:06:24.3245443Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=query-frontend waiting_for=usage-stats
ts=2023-03-21T18:06:24.3245515Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=query-frontend waiting_for=activity-tracker
ts=2023-03-21T18:06:24.32455646Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=query-frontend waiting_for=memberlist-kv
ts=2023-03-21T18:06:24.32457166Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=memberlist-kv waiting_for=usage-stats
ts=2023-03-21T18:06:24.32457778Z caller=module_service.go:82 level=info msg=initialising module=memberlist-kv
ts=2023-03-21T18:06:24.3245927Z caller=module_service.go:82 level=info msg=initialising module=query-frontend
ts=2023-03-21T18:06:24.32462434Z caller=module_service.go:82 level=info msg=initialising module=compactor
ts=2023-03-21T18:06:24.32460042Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=store-queryable
ts=2023-03-21T18:06:24.32464662Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ingester-service waiting_for=sanity-check
ts=2023-03-21T18:06:24.32465462Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ingester-service waiting_for=server
ts=2023-03-21T18:06:24.32466018Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ingester-service waiting_for=usage-stats
ts=2023-03-21T18:06:24.3246717Z caller=module_service.go:82 level=info msg=initialising module=ingester-service
ts=2023-03-21T18:06:24.32471094Z caller=ingester.go:1831 level=info msg="opening existing TSDBs"
ts=2023-03-21T18:06:24.324883381Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=distributor-service waiting_for=ring
ts=2023-03-21T18:06:24.324894141Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ring waiting_for=sanity-check
ts=2023-03-21T18:06:24.324899421Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ring waiting_for=server
ts=2023-03-21T18:06:24.324904301Z caller=module_service.go:82 level=info msg=initialising module=ring
ts=2023-03-21T18:06:24.324991021Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-gateway waiting_for=sanity-check
ts=2023-03-21T18:06:24.324998301Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-gateway waiting_for=server
ts=2023-03-21T18:06:24.325003301Z caller=module_service.go:82 level=info msg=initialising module=store-gateway
ts=2023-03-21T18:06:24.325038341Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-queryable waiting_for=sanity-check
ts=2023-03-21T18:06:24.325044021Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-queryable waiting_for=server
ts=2023-03-21T18:06:24.325048661Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=store-queryable waiting_for=usage-stats
ts=2023-03-21T18:06:24.325053461Z caller=module_service.go:82 level=info msg=initialising module=store-queryable
ts=2023-03-21T18:06:24.325444262Z caller=repair.go:56 level=info user=anonymous msg="Found healthy block" mint=1679381823938 maxt=1679381823939 ulid=01GW1JV3BAWKZZQSXPN2C6PYKH
ts=2023-03-21T18:06:24.325509582Z caller=repair.go:56 level=info user=anonymous msg="Found healthy block" mint=1679381823938 maxt=1679381823939 ulid=01GW1KJYP98JKHRNJRTXJRD50K
ts=2023-03-21T18:06:24.331126919Z caller=db.go:1417 level=debug user=anonymous msg="Overlapping blocks found during reloadBlocks" detail="[mint: 1679381823938, maxt: 1679381823939, range: 0s, blocks: 2]: <ulid: 01GW1JV3BAWKZZQSXPN2C6PYKH, mint: 1679381823938, maxt: 1679381823939, range: 0s>, <ulid: 01GW1KJYP98JKHRNJRTXJRD50K, mint: 1679381823938, maxt: 1679381823939, range: 0s>"
ts=2023-03-21T18:06:24.331167319Z caller=head.go:591 level=info user=anonymous msg="Replaying on-disk memory mappable chunks if any"
ts=2023-03-21T18:06:24.331180279Z caller=head.go:635 level=info user=anonymous msg="On-disk memory mappable chunks replay completed" duration=2.92µs
ts=2023-03-21T18:06:24.331187919Z caller=head.go:641 level=info user=anonymous msg="Replaying WAL, this may take a while"
ts=2023-03-21T18:06:24.334397049Z caller=head.go:677 level=info user=anonymous msg="WAL checkpoint loaded"
ts=2023-03-21T18:06:24.334585209Z caller=head.go:712 level=info user=anonymous msg="WAL segment loaded" segment=13 maxSegment=16
ts=2023-03-21T18:06:24.33482457Z caller=head.go:712 level=info user=anonymous msg="WAL segment loaded" segment=14 maxSegment=16
ts=2023-03-21T18:06:24.33493841Z caller=head.go:712 level=info user=anonymous msg="WAL segment loaded" segment=15 maxSegment=16
ts=2023-03-21T18:06:24.335027171Z caller=head.go:712 level=info user=anonymous msg="WAL segment loaded" segment=16 maxSegment=16
ts=2023-03-21T18:06:24.335044371Z caller=head.go:749 level=info user=anonymous msg="WAL replay completed" checkpoint_replay_duration=3.23121ms wal_replay_duration=616.162µs wbl_replay_duration=160ns total_replay_duration=3.865932ms
ts=2023-03-21T18:06:24.336858976Z caller=db.go:1746 level=info user=anonymous msg="Compactions disabled"
ts=2023-03-21T18:06:24.336885536Z caller=ingester.go:1750 level=info user=anonymous msg="Running compaction after WAL replay"
ts=2023-03-21T18:06:24.348617731Z caller=compact.go:700 level=info user=anonymous msg="write block resulted in empty block" mint=1679381823939 maxt=1679385600000 duration=11.715715ms
ts=2023-03-21T18:06:24.348766771Z caller=db.go:1417 level=debug user=anonymous msg="Overlapping blocks found during reloadBlocks" detail="[mint: 1679381823938, maxt: 1679381823939, range: 0s, blocks: 2]: <ulid: 01GW1JV3BAWKZZQSXPN2C6PYKH, mint: 1679381823938, maxt: 1679381823939, range: 0s>, <ulid: 01GW1KJYP98JKHRNJRTXJRD50K, mint: 1679381823938, maxt: 1679381823939, range: 0s>"
ts=2023-03-21T18:06:24.349832174Z caller=head.go:1246 level=info user=anonymous msg="Head GC completed" caller=truncateMemory duration=1.045563ms
ts=2023-03-21T18:06:24.349920735Z caller=checkpoint.go:100 level=info user=anonymous msg="Creating checkpoint" from_segment=13 to_segment=14 mint=1679385600000
ts=2023-03-21T18:06:24.355729872Z caller=head.go:1218 level=info user=anonymous msg="WAL checkpoint complete" first=13 last=14 duration=5.868578ms
ts=2023-03-21T18:06:24.355971592Z caller=ingester.go:1927 level=info msg="successfully opened existing TSDBs"
ts=2023-03-21T18:06:24.356013273Z caller=lifecycler.go:566 level=info msg="not loading tokens from file, tokens file path is empty"
ts=2023-03-21T18:06:38.635250479Z caller=ingester.go:2128 level=debug msg="TSDB blocks compaction completed successfully" user=anonymous compactReason=regular
ts=2023-03-21T18:06:40.746110592Z caller=log.go:194 level=debug msg="Stream connection from=10.0.17.14:35612"
ts=2023-03-21T18:06:40.760928676Z caller=resolver.go:87 level=error msg="failed to lookup IP addresses" host=tasks.monitoring_mimir err="lookup tasks.monitoring_mimir on 127.0.0.11:53: no such host"
ts=2023-03-21T18:06:40.760977956Z caller=resolver.go:134 level=warn msg="IP address lookup yielded no results. No host found or no addresses found" host=tasks.monitoring_mimir
ts=2023-03-21T18:06:40.760994036Z caller=memberlist_client.go:543 level=info msg="memberlist fast-join starting" nodes_found=0 to_join=0
ts=2023-03-21T18:06:40.761007356Z caller=memberlist_client.go:563 level=warn msg="memberlist fast-join finished" joined_nodes=0 elapsed_time=25.078844737s
ts=2023-03-21T18:06:40.761027116Z caller=memberlist_client.go:576 level=info msg="joining memberlist cluster" join_members=dns+tasks.monitoring_mimir:7946
ts=2023-03-21T18:06:40.761183556Z caller=ring.go:263 level=info component=compactor msg="ring doesn't exist in KV store yet"
ts=2023-03-21T18:06:40.761210436Z caller=ring.go:263 level=info msg="ring doesn't exist in KV store yet"
ts=2023-03-21T18:06:40.761242436Z caller=ring.go:263 level=info msg="ring doesn't exist in KV store yet"
ts=2023-03-21T18:06:40.761279797Z caller=lifecycler.go:595 level=info msg="instance not found in ring, adding with no tokens" ring=ingester
ts=2023-03-21T18:06:40.761363757Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=usage-stats
ts=2023-03-21T18:06:40.761378477Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=activity-tracker
ts=2023-03-21T18:06:40.761386877Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=distributor-service
ts=2023-03-21T18:06:40.761194636Z caller=ring.go:263 level=info msg="ring doesn't exist in KV store yet"
ts=2023-03-21T18:06:40.761309597Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=usage-stats
ts=2023-03-21T18:06:40.761471517Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=activity-tracker
ts=2023-03-21T18:06:40.761480917Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=distributor-service
ts=2023-03-21T18:06:40.761304117Z caller=module_service.go:82 level=info msg=initialising module=distributor-service
ts=2023-03-21T18:06:40.761529797Z caller=basic_lifecycler.go:261 level=info component=compactor msg="instance not found in the ring" instance=be7c11639b37 ring=compactor
ts=2023-03-21T18:06:40.761569157Z caller=basic_lifecycler.go:261 level=info msg="instance not found in the ring" instance=be7c11639b37 ring=distributor
ts=2023-03-21T18:06:40.761687438Z caller=lifecycler.go:431 level=debug msg="JoinAfter expired" ring=ingester
ts=2023-03-21T18:06:40.761702838Z caller=lifecycler.go:435 level=info msg="auto-joining cluster after timeout" ring=ingester
ts=2023-03-21T18:06:40.761724158Z caller=distributor.go:498 level=info msg="waiting until distributor is ACTIVE in the ring"
ts=2023-03-21T18:06:40.761761998Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
ts=2023-03-21T18:06:40.761770238Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=ring
ts=2023-03-21T18:06:40.761775638Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=querier waiting_for=sanity-check
ts=2023-03-21T18:06:40.761781478Z caller=module_service.go:82 level=info msg=initialising module=querier
ts=2023-03-21T18:06:40.761756558Z caller=basic_lifecycler.go:261 level=info msg="instance not found in the ring" instance=be7c11639b37 ring=store-gateway
ts=2023-03-21T18:06:40.761816438Z caller=basic_lifecycler_delegates.go:63 level=info msg="not loading tokens from file, tokens file path is empty"
ts=2023-03-21T18:06:40.761823238Z caller=worker.go:232 level=info msg="adding connection" addr=127.0.0.1:9095 in-use=true
ts=2023-03-21T18:06:40.761949519Z caller=memberlist_client.go:883 level=debug msg="CAS attempt failed" err="no change detected" retry=true
ts=2023-03-21T18:06:40.762083759Z caller=compactor.go:430 level=info component=compactor msg="waiting until compactor is ACTIVE in the ring"
ts=2023-03-21T18:06:40.762179199Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=memberlist-kv
ts=2023-03-21T18:06:40.762191839Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=ring
ts=2023-03-21T18:06:40.762197799Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=sanity-check
ts=2023-03-21T18:06:40.762203319Z caller=module_service.go:72 level=debug msg="module waiting for initialization" module=ruler waiting_for=server
ts=2023-03-21T18:06:40.762239879Z caller=module_service.go:82 level=info msg=initialising module=ruler
ts=2023-03-21T18:06:40.762979882Z caller=basic_lifecycler.go:261 level=info msg="instance not found in the ring" instance=be7c11639b37 ring=ruler
ts=2023-03-21T18:06:40.763173082Z caller=ruler.go:347 level=info msg="waiting until ruler is ACTIVE in the ring"
ts=2023-03-21T18:06:40.763183962Z caller=ruler.go:351 level=info msg="ruler is ACTIVE in the ring"
ts=2023-03-21T18:06:40.763301043Z caller=ruler.go:399 level=info msg="ruler up and running"
ts=2023-03-21T18:06:40.763310603Z caller=ruler.go:433 level=debug msg="syncing rules" reason=initial
ts=2023-03-21T18:06:40.764621846Z caller=gateway.go:199 level=info msg="waiting until store-gateway is JOINING in the ring"
ts=2023-03-21T18:06:40.764946967Z caller=gateway.go:203 level=info msg="store-gateway is JOINING in the ring"
ts=2023-03-21T18:06:40.764988328Z caller=bucket_stores.go:169 level=info msg="synchronizing TSDB blocks for all users"
ts=2023-03-21T18:06:40.768019897Z caller=log.go:194 level=debug msg="Initiating push/pull sync with: 10.0.17.15:7946"
ts=2023-03-21T18:06:40.76915402Z caller=log.go:194 level=debug msg="Stream connection from=10.0.17.15:40916"
ts=2023-03-21T18:06:40.771064945Z caller=log.go:194 level=debug msg="Initiating push/pull sync with: 10.0.17.14:7946"
ts=2023-03-21T18:06:40.772882951Z caller=memberlist_client.go:595 level=info msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=11.855435ms
ts=2023-03-21T18:06:40.896932157Z caller=compactor.go:435 level=info component=compactor msg="compactor is ACTIVE in the ring"
ts=2023-03-21T18:06:40.897318038Z caller=compactor.go:556 level=info component=compactor msg="discovering users from bucket"
ts=2023-03-21T18:06:40.897355638Z caller=blocks_cleaner.go:192 level=info component=cleaner run_id=1679422000 msg="started blocks cleanup and maintenance"
ts=2023-03-21T18:06:41.762422153Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/ring version=2 oldVersion=1 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:06:42.883351943Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:44.845905418Z caller=compactor.go:566 level=info component=compactor msg="discovered users from bucket" users=1
ts=2023-03-21T18:06:44.857601293Z caller=blocks_cleaner.go:361 level=info component=cleaner user=anonymous msg="started blocks cleanup and maintenance"
ts=2023-03-21T18:06:44.857679373Z caller=bucket_stores.go:422 level=info user=anonymous msg="creating user bucket store"
ts=2023-03-21T18:06:44.857932934Z caller=compactor.go:608 level=info component=compactor msg="starting compaction of user blocks" user=anonymous
ts=2023-03-21T18:06:44.858196615Z caller=bucket_compactor.go:791 level=info component=compactor user=anonymous msg="start sync of metas"
ts=2023-03-21T18:06:44.858234655Z caller=fetcher.go:324 level=debug component=compactor user=anonymous component=block.BaseFetcher msg="fetching meta data" concurrency=20
ts=2023-03-21T18:06:44.873090179Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GW1JV3BAWKZZQSXPN2C6PYKH
ts=2023-03-21T18:06:44.873136779Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GW1JXYXDHN3KEM5RR5G7W3Y9
ts=2023-03-21T18:06:44.873142619Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GW1JV3BAWKZZQSXPN2C6PYKH/index-header
ts=2023-03-21T18:06:44.873169499Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GW1JXYXDHN3KEM5RR5G7W3Y9/index-header
ts=2023-03-21T18:06:44.873212859Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GW1KJYP98JKHRNJRTXJRD50K
ts=2023-03-21T18:06:44.873231739Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GW1KJYP98JKHRNJRTXJRD50K/index-header
ts=2023-03-21T18:06:44.873289979Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GVEV679Z48336KC6X70WCFF6
ts=2023-03-21T18:06:44.873338059Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GVMXN339R458KVFSAKPMFG08
ts=2023-03-21T18:06:44.873360179Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GVMXN339R458KVFSAKPMFG08/index-header
ts=2023-03-21T18:06:44.87347402Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GVJDHRVAD0J3V1BWSTTBBSGV
ts=2023-03-21T18:06:44.873310219Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GVEV679Z48336KC6X70WCFF6/index-header
ts=2023-03-21T18:06:44.87349666Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GVJDHRVAD0J3V1BWSTTBBSGV/index-header
ts=2023-03-21T18:06:44.87366762Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GVC9V826EG92KG7D6SP9JTDW
ts=2023-03-21T18:06:44.87371718Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GV73RTDZW50EPSNQFYG9PQW6
ts=2023-03-21T18:06:44.873768461Z caller=bucket.go:403 level=debug user=anonymous msg="loading new block" id=01GV9RG9BQWM8RD2R0N3KQ4YSZ
ts=2023-03-21T18:06:44.8737077Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GVC9V826EG92KG7D6SP9JTDW/index-header
ts=2023-03-21T18:06:44.873765461Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GV73RTDZW50EPSNQFYG9PQW6/index-header
ts=2023-03-21T18:06:44.874002461Z caller=lazy_binary_reader.go:108 level=debug user=anonymous msg="the index-header doesn't exist on disk; recreating" path=tsdb-sync/anonymous/01GV9RG9BQWM8RD2R0N3KQ4YSZ/index-header
ts=2023-03-21T18:06:44.884552492Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:45.167736769Z caller=blocks_cleaner.go:366 level=info component=cleaner user=anonymous msg="completed blocks cleanup and maintenance" duration=310.134476ms
ts=2023-03-21T18:06:45.167784809Z caller=blocks_cleaner.go:198 level=info component=cleaner run_id=1679422000 msg="successfully completed blocks cleanup and maintenance"
ts=2023-03-21T18:06:45.170084896Z caller=fetcher.go:474 level=info component=compactor user=anonymous component=block.BaseFetcher msg="successfully synchronized block metadata" duration=311.868081ms duration_ms=311 cached=11 returned=9 partial=0
ts=2023-03-21T18:06:45.170117016Z caller=bucket_compactor.go:796 level=info component=compactor user=anonymous msg="start of GC"
ts=2023-03-21T18:06:45.170184856Z caller=bucket_compactor.go:837 level=info component=compactor user=anonymous msg="start of compactions"
ts=2023-03-21T18:06:45.170204256Z caller=bucket_compactor.go:874 level=info component=compactor user=anonymous msg="compaction iterations done"
ts=2023-03-21T18:06:45.170332696Z caller=compactor.go:625 level=info component=compactor msg="successfully compacted user blocks" user=anonymous
ts=2023-03-21T18:06:45.370366007Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GW1JV3BAWKZZQSXPN2C6PYKH/index-header elapsed=497.198628ms
ts=2023-03-21T18:06:45.370419967Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=497.332548ms id=01GW1JV3BAWKZZQSXPN2C6PYKH
ts=2023-03-21T18:06:45.503227559Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GVMXN339R458KVFSAKPMFG08/index-header elapsed=629.85074ms
ts=2023-03-21T18:06:45.503287479Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=629.94774ms id=01GVMXN339R458KVFSAKPMFG08
ts=2023-03-21T18:06:45.523319859Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GW1JXYXDHN3KEM5RR5G7W3Y9/index-header elapsed=650.13092ms
ts=2023-03-21T18:06:45.523369939Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=650.2356ms id=01GW1JXYXDHN3KEM5RR5G7W3Y9
ts=2023-03-21T18:06:45.533327608Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GW1KJYP98JKHRNJRTXJRD50K/index-header elapsed=660.082309ms
ts=2023-03-21T18:06:45.533367088Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=660.155829ms id=01GW1KJYP98JKHRNJRTXJRD50K
ts=2023-03-21T18:06:45.546451567Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GVJDHRVAD0J3V1BWSTTBBSGV/index-header elapsed=672.938067ms
ts=2023-03-21T18:06:45.546498927Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=673.025267ms id=01GVJDHRVAD0J3V1BWSTTBBSGV
ts=2023-03-21T18:06:45.55781752Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GVEV679Z48336KC6X70WCFF6/index-header elapsed=684.3127ms
ts=2023-03-21T18:06:45.557866921Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=684.574182ms id=01GVEV679Z48336KC6X70WCFF6
ts=2023-03-21T18:06:45.575768933Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GV73RTDZW50EPSNQFYG9PQW6/index-header elapsed=701.925592ms
ts=2023-03-21T18:06:45.575811134Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=702.094794ms id=01GV73RTDZW50EPSNQFYG9PQW6
ts=2023-03-21T18:06:45.703717351Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GV9RG9BQWM8RD2R0N3KQ4YSZ/index-header elapsed=829.68801ms
ts=2023-03-21T18:06:45.703770431Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=830.00289ms id=01GV9RG9BQWM8RD2R0N3KQ4YSZ
ts=2023-03-21T18:06:45.71685299Z caller=lazy_binary_reader.go:115 level=debug user=anonymous msg="built index-header file" path=tsdb-sync/anonymous/01GVC9V826EG92KG7D6SP9JTDW/index-header elapsed=843.037169ms
ts=2023-03-21T18:06:45.71691327Z caller=bucket.go:412 level=info user=anonymous msg="loaded new block" elapsed=843.24705ms id=01GVC9V826EG92KG7D6SP9JTDW
ts=2023-03-21T18:06:45.717040631Z caller=bucket_stores.go:178 level=info msg="successfully synchronized TSDB blocks for all users"
ts=2023-03-21T18:06:45.717311031Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/store-gateway version=2 oldVersion=1 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:06:45.717343952Z caller=gateway.go:238 level=info msg="waiting until store-gateway is ACTIVE in the ring"
ts=2023-03-21T18:06:45.91671574Z caller=gateway.go:242 level=info msg="store-gateway is ACTIVE in the ring"
ts=2023-03-21T18:06:45.9167913Z caller=mimir.go:792 level=info msg="Application started"
ts=2023-03-21T18:06:46.886225443Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:48.886905271Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:50.888293061Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:52.683607162Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:52.88922901Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:54.684692711Z caller=log.go:194 level=debug msg="Failed UDP ping: 7b0b6d4157ca-f37acca4 (timeout reached)"
ts=2023-03-21T18:06:55.6832459Z caller=log.go:194 level=info msg="Suspect 7b0b6d4157ca-f37acca4 has failed, no acks received"
ts=2023-03-21T18:06:57.684246409Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:57.884159759Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:06:59.684776716Z caller=log.go:194 level=debug msg="Failed UDP ping: 7b0b6d4157ca-f37acca4 (timeout reached)"
ts=2023-03-21T18:06:59.884765507Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:01.885382494Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:03.886481643Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:05.68359563Z caller=log.go:194 level=info msg="Suspect 7b0b6d4157ca-f37acca4 has failed, no acks received"
ts=2023-03-21T18:07:12.683709781Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:12.884201133Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:14.684106008Z caller=log.go:194 level=debug msg="Failed UDP ping: 7b0b6d4157ca-f37acca4 (timeout reached)"
ts=2023-03-21T18:07:14.88469224Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:15.683674239Z caller=log.go:194 level=info msg="Marking 7b0b6d4157ca-f37acca4 as failed, suspect timeout reached (0 peer confirmations)"
ts=2023-03-21T18:07:16.885381268Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:18.886788218Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:20.887677806Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:22.888989636Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:24.337163392Z caller=db.go:1417 level=debug user=anonymous msg="Overlapping blocks found during reloadBlocks" detail="[mint: 1679381823938, maxt: 1679381823939, range: 0s, blocks: 2]: <ulid: 01GW1JV3BAWKZZQSXPN2C6PYKH, mint: 1679381823938, maxt: 1679381823939, range: 0s>, <ulid: 01GW1KJYP98JKHRNJRTXJRD50K, mint: 1679381823938, maxt: 1679381823939, range: 0s>"
ts=2023-03-21T18:07:24.889851944Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:25.683779288Z caller=log.go:194 level=info msg="Suspect 7b0b6d4157ca-f37acca4 has failed, no acks received"
ts=2023-03-21T18:07:27.883298343Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:28.591868795Z caller=ingester.go:2028 level=debug msg="shipper successfully synchronized TSDB blocks with storage" user=anonymous uploaded=0
ts=2023-03-21T18:07:29.88371025Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:31.88513268Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:33.886178349Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:38.635683453Z caller=ingester.go:2128 level=debug msg="TSDB blocks compaction completed successfully" user=anonymous compactReason=regular
ts=2023-03-21T18:07:40.763396176Z caller=ruler.go:433 level=debug msg="syncing rules" reason=periodic
ts=2023-03-21T18:07:42.884040278Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:44.885201427Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:46.885713694Z caller=tcp_transport.go:428 level=warn component="memberlist TCPTransport" msg="WriteTo failed" addr=192.168.192.3:7946 err="dial tcp 192.168.192.3:7946: i/o timeout"
ts=2023-03-21T18:07:55.762819347Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/distributor version=6 oldVersion=5 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:07:55.762878867Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/ring version=7 oldVersion=6 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:07:55.762915307Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/compactor version=6 oldVersion=5 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:07:55.76386463Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=rulers/ring version=6 oldVersion=5 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:07:55.764930473Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/store-gateway version=7 oldVersion=6 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:08:10.761954276Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/distributor version=7 oldVersion=6 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:08:10.762015276Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/compactor version=7 oldVersion=6 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:08:10.762087317Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/ring version=8 oldVersion=7 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:08:10.764243203Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=rulers/ring version=7 oldVersion=6 content=[be7c11639b37] oldContent=[be7c11639b37]
ts=2023-03-21T18:08:10.765298846Z caller=broadcast.go:48 level=debug msg="Invalidating forwarded broadcast" key=collectors/store-gateway version=8 oldVersion=7 content=[be7c11639b37] oldContent=[be7c11639b37]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment