Skip to content

Instantly share code, notes, and snippets.

@amckinley
Created February 1, 2021 00:03
Show Gist options
  • Save amckinley/eea8e5617cdf687524cca326670b78b9 to your computer and use it in GitHub Desktop.
Save amckinley/eea8e5617cdf687524cca326670b78b9 to your computer and use it in GitHub Desktop.
level=warn ts=2021-01-31T23:53:11.343049641Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EXAHJGKSGFTQ75X5FNNPP3SK:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343138289Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=S:01EX9P3FDZ055JCKZ7ZS22ECMN:41603280 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343198509Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX9P34379RNTDWJMJT3KC8H4:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343225582Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EXARD9S52S833EDTV1N4PNM6:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211"
level=warn ts=2021-01-31T23:53:11.343228637Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EXARCPWGCY1ZEFYM92ZW7RC6:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343260751Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX91G2356WDQ8W550ESD8F1J:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343286865Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX9F67HNVEJASGS6D2NXY000:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343297522Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX9F7493XY5PF8MD5E3D7A0B:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343250871Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX9P34FVZRA87M87JCC2TJ75:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211"
level=warn ts=2021-01-31T23:53:11.343348809Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EXAHHZ55RKBK2B7S0829S6JP:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211"
level=warn ts=2021-01-31T23:53:11.343393875Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=7 firstKey=P:01EX98C4PQHSAWQPJEDMY95H46:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211"
level=warn ts=2021-01-31T23:53:11.343432426Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX98B4TJTCSPNNZ8ZPMV7PZS:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211"
level=warn ts=2021-01-31T23:53:11.343470569Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EXAZ9E6BQJ6A81XWHJQAZEP1:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211"
level=warn ts=2021-01-31T23:53:11.343462462Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX9F6J47GDF5D3G0TG4B8GWA:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211"
level=info ts=2021-01-31T23:53:11.376494137Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=20.803172828s id=01ESSEX8EDA4STFQV1JJD7EMR5
level=warn ts=2021-01-31T23:53:11.390090352Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=242.504939ms err="rpc error: code = Aborted desc = fetch series for block 01EX1R0NTGXESM004QECZJJ8GC: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.956959115Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957147646Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957149678Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.95728205Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957296701Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957361595Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957365621Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957363421Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX8CVWNECCE7BFKP2K6JAV2A:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:11.957621888Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=807.675587ms err="rpc error: code = Aborted desc = fetch series for block 01EX61BF90G6YSHK77S0C9A9WJ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957631992Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.32530398s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957627058Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=18.762633565s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957656695Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694562044s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.95777634Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=18.789148222s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957803274Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.281748961s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957832084Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.282431546s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957847276Z caller=grpc_logging.go:55 duration=18.762546676s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957855371Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.695489724s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957867812Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=807.850912ms err="rpc error: code = Aborted desc = fetch series for block 01EX5TGKRTC4CH9FAMT8QJQTM4: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957868609Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694784205s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957875221Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=819.712489ms err="rpc error: code = Aborted desc = fetch series for block 01EX8KQAPJWAXG2QW7GJ6W7KPC: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.957891925Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=11.105779437s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.958031279Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694568108s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.958048866Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.695648799s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:11.958054201Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.694870885s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.368025466Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:12.36805452Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context deadline exceeded"
level=warn ts=2021-01-31T23:53:12.368079157Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context deadline exceeded"
level=warn ts=2021-01-31T23:53:12.36817648Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVAKB2P94Y7MG1GTRYQD4TNR:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:12.368285483Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m33.873592935s err="rpc error: code = Aborted desc = fetch series for block 01EV93950R0B8D15VJCEXVPM27: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.368449431Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=2m4.050955267s err="rpc error: code = Aborted desc = fetch series for block 01EV939W5K31H36XH9PC4Z0HZF: expanded matching posting: get postings: read postings range: read range: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.36844958Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m27.047762389s err="rpc error: code = Aborted desc = fetch series for block 01EV9A439HJAKPZ0JJDGZBZDYQ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.368472593Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=3m28.442545424s err="rpc error: code = Aborted desc = fetch series for block 01EV939W5K31H36XH9PC4Z0HZF: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.36856347Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m3.974794111s err="rpc error: code = Aborted desc = fetch series for block 01EV80XRBJ9WVTV9YH54VCXGZS: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.369077641Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m18.661856529s err="rpc error: code = Aborted desc = fetch series for block 01EV8ENZQD6D1Z48NKHX8J6SHD: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.429636435Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVS68N7C7T5RW7XQ5A07M44C:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:12.429893942Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m29.314522431s err="rpc error: code = Aborted desc = fetch series for block 01EVQFAFWZP1MJ4Z19SMS08271: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.42997345Z caller=grpc_logging.go:55 duration=4m33.928759114s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EVQFAFWZP1MJ4Z19SMS08271: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.430102172Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m4.03314589s err="rpc error: code = Aborted desc = fetch series for block 01EVQP6MRD00T92DXGRFJWCQKX: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.430811972Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m4.033891552s err="rpc error: code = Aborted desc = fetch series for block 01EVQP6MRD00T92DXGRFJWCQKX: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.444550963Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EX5KK5RF994VZEKPD4Y1CX9G:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:12.444611736Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.771197618s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445102977Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.276439592s err="rpc error: code = Aborted desc = fetch series for block 01EX8CVWNECCE7BFKP2K6JAV2A: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445462757Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.801971193s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445610833Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=20.183118048s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445609705Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=20.183138967s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445635043Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=19.250272442s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445732312Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=20.182729644s err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.445731068Z caller=grpc_logging.go:55 duration=20.182797936s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EX5KK5RF994VZEKPD4Y1CX9G: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.524130631Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m27.307749729s err="rpc error: code = Aborted desc = fetch series for block 01EVYB2AHNA5KY6AETVAG7TC4W: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.524222444Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m34.027191935s err="rpc error: code = Aborted desc = fetch series for block 01EVXXAK4C8Y95A19X4YT8WF1Q: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.966626754Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m34.472055433s err="rpc error: code = Aborted desc = fetch series for block 01EV6QRT1C9AVYVV716F7ABK8K: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:12.966744099Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m28.986864233s err="rpc error: code = Aborted desc = fetch series for block 01EV6QRT1C9AVYVV716F7ABK8K: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n"
level=info ts=2021-01-31T23:53:12.974596472Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=24.051961462s id=01ETQ2GTR4G87CZDCSDB8T09W1
level=warn ts=2021-01-31T23:53:13.471464729Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m30.342884486s err="rpc error: code = Aborted desc = fetch series for block 01EVHWSYD41MS3ZDBA2A7TMF18: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:13.471460547Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m34.976754391s err="rpc error: code = Aborted desc = fetch series for block 01EVHWSYD41MS3ZDBA2A7TMF18: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:13.471507787Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=3m46.42793758s err="rpc error: code = Aborted desc = fetch series for block 01EVFZ0P60KPQPWA4FZTT55XA6: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:13.489283002Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m12.010613701s err="rpc error: code = Aborted desc = fetch series for block 01EVGKJQ2HN118ENPB1292X5QN: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:14.196148666Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV3SC6K3SNZ5BWDCN6M03V:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:14.196247146Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV3SC6K3SNZ5BWDCN6M03V:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context deadline exceeded"
level=warn ts=2021-01-31T23:53:14.221601519Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m20.513910043s err="rpc error: code = Aborted desc = fetch series for block 01EVM8BJSB72C45NCV6XSX2H1M: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:14.221598817Z caller=grpc_logging.go:55 duration=4m28.964798359s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EVJAGST35SFJ7SH6GWR40JZN: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:14.221634747Z caller=grpc_logging.go:55 duration=4m35.727079755s method=/gatewaypb.StoreGateway/Series err="rpc error: code = Aborted desc = fetch series for block 01EVJAGST35SFJ7SH6GWR40JZN: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:14.597463325Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=2m21.116840928s err="rpc error: code = Aborted desc = fetch series for block 01EV0GM3R428SGPDEM4VRTMZ9E: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:15.564493246Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVENSB46YZVQA71QBG0WQS8K:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:15.604835234Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m37.109816104s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:15.60485295Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=2m22.12426968s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:15.604857486Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=1m7.210766342s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:15.604853075Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m30.3882386s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:15.604941305Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV4X2SH4JVME81NT5BJGMA:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="failed to wait for turn: context canceled"
level=warn ts=2021-01-31T23:53:15.605278644Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m21.895793234s err="rpc error: code = Aborted desc = fetch series for block 01EW1RY6GNHPDJMRTACF9SFF84: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:15.805887821Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EVZV4X2SH4JVME81NT5BJGMA:tV-cEZObnRBJX6mvyPv_gi-z5eXOLwQvYZAa64YL2Tg err="read tcp 10.11.134.125:48666->10.11.105.37:11211: i/o timeout"
level=warn ts=2021-01-31T23:53:15.806459803Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=3m31.880567966s err="rpc error: code = Aborted desc = fetch series for block 01EW0XEGE3NP7M550X1ASAKF2Y: expanded matching posting: get postings: read postings range: get range reader: context deadline exceeded" msg="gRPC\n"
level=info ts=2021-01-31T23:53:16.368519122Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.850008854s id=01EWN2XC88AF6BNJVAZK3FQEHA
level=info ts=2021-01-31T23:53:16.370621093Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.871352225s id=01EW1B5N726XJ01V8RKBGJ3WKA
level=info ts=2021-01-31T23:53:16.372195988Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.1733649s id=01EVR3YN9BRV19TKEWK31YSK4A
level=info ts=2021-01-31T23:53:16.373583139Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.854518142s id=01ESS15PM6QM0KWSMWVKT2MKC2
level=info ts=2021-01-31T23:53:16.375052299Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.189782239s id=01EXAZ9F7C5H5JNCW5CHESTQEH
level=info ts=2021-01-31T23:53:16.376429517Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.174933994s id=01ETSMWHZFPRNTNHSY4VS0WKJQ
level=info ts=2021-01-31T23:53:16.377959104Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.176485971s id=01EWPSV69JHTHMZXCC7FKBD046
level=info ts=2021-01-31T23:53:16.37953622Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.197427573s id=01ERM45BDSFDPJEHKEHF6S2320
level=info ts=2021-01-31T23:53:16.462119558Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.944370371s id=01ESWNYAS0FZ10ZRYPQ8QWC2KS
level=info ts=2021-01-31T23:53:16.463606979Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.264748886s id=01EWT0SZYH058F5VAKV0ZZ4YQ4
level=info ts=2021-01-31T23:53:16.465615711Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.266803313s id=01ERVMFMGFQ9047AMQFZ8PCX0Q
level=info ts=2021-01-31T23:53:16.466969538Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.949567432s id=01EVM1F2SMHDS7HAAT2S88PEXZ
level=info ts=2021-01-31T23:53:16.468466576Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.971635469s id=01EVJHC9DAGP3VEAV809T9CBSR
level=info ts=2021-01-31T23:53:16.46995367Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.95093276s id=01EW9G3VD1RH1H884SQV81VBJ3
level=info ts=2021-01-31T23:53:16.471370434Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.267142161s id=01EVNRD1G48N1X19M5NPNMFPG8
level=info ts=2021-01-31T23:53:16.472806998Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.953985647s id=01EVWM3EPHZ3N4543KH26C5WRD
level=info ts=2021-01-31T23:53:16.474242425Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=9.981087583s id=01EVCQZNNA13EV2XYDRQ6BM35R
level=warn ts=2021-01-31T23:53:16.533088804Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m38.037829014s err="rpc error: code = Aborted desc = fetch series for block 01EVPCZVMZGYJNZH31TN5GSM58: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:16.702699162Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m22.993018585s err="rpc error: code = Aborted desc = fetch series for block 01EV4SYEY94X7X8VV9H15M7WM2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=warn ts=2021-01-31T23:53:16.703343633Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=4m38.208539352s err="rpc error: code = Aborted desc = fetch series for block 01EV1C3BTTTJC9ZCGQYCMJ7EE2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n"
level=info ts=2021-01-31T23:53:16.739538771Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=5.363009721s id=01ERZPYP84QZZMBVTCW2T6V32M
level=info ts=2021-01-31T23:53:16.888458112Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.686983992s id=01ERS2360PXK9MW97ZV4N5ZC0E
unexpected fault address 0x7f4c11e8dc72
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f4c11e8dc72 pc=0x40205f]
goroutine 4498534 [running]:
runtime.throw(0x2533220, 0x5)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc13e42cf38 sp=0xc13e42cf08 pc=0x434de2
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:702 +0x3cc fp=0xc13e42cf68 sp=0xc13e42cf38 pc=0x44bb8c
cmpbody()
/usr/local/go/src/internal/bytealg/compare_amd64.s:84 +0x9f fp=0xc13e42cf70 sp=0xc13e42cf68 pc=0x40205f
github.com/thanos-io/thanos/pkg/block/indexheader.BinaryReader.postingsOffset(0x2b73f00, 0xc1e84a0760, 0xc162eed9d0, 0x2b44ce0, 0xc1e84a0740, 0xc19d6c31a0, 0x0, 0xc1f2794d40, 0xc2574b6d20, 0xc9e3ffeed0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:696 +0xe54 fp=0xc13e42d0e8 sp=0xc13e42cf70 pc=0x18f5b94
github.com/thanos-io/thanos/pkg/block/indexheader.BinaryReader.PostingsOffset(0x2b73f00, 0xc1e84a0760, 0xc162eed9d0, 0x2b44ce0, 0xc1e84a0740, 0xc19d6c31a0, 0x0, 0xc1f2794d40, 0xc2574b6d20, 0xc9e3ffeed0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:646 +0xbf fp=0xc13e42d1c8 sp=0xc13e42d0e8 pc=0x18f4b6f
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c418630, 0xc17f393f19, 0x3, 0x7f4c11e8dc72, 0xe, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:162 +0x1b3 fp=0xc13e42d318 sp=0xc13e42d1c8 pc=0x18f7db3
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1ec83cec0, 0xc0522db040, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 fp=0xc13e42d578 sp=0xc13e42d318 pc=0x199db71
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1ec83cec0, 0xc1e8721b60, 0x2, 0x2, 0x216f320, 0x3f99388, 0x0, 0x0, 0xbffdee5d4730877c)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac fp=0xc13e42d818 sp=0xc13e42d578 pc=0x199b68c
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc004d1b1a0, 0xc1ec83cec0, 0xc1d7a5ed80, 0xc1e8721b60, 0x2, 0x2, 0xc1ea4afe60, 0x2b44f80, 0xc217e2e2d0, 0xc00017c0b8, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a fp=0xc13e42dd00 sp=0xc13e42d818 pc=0x19931ba
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc08f8eaa20, 0xc1eb7e68c0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe fp=0xc13e42df78 sp=0xc13e42dd00 pc=0x19a373e
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc217e2e2a0, 0xc23e3532d0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 fp=0xc13e42dfd0 sp=0xc13e42df78 pc=0x997bc9
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc13e42dfd8 sp=0xc13e42dfd0 pc=0x467ca1
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 1 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AwaitStopped(0xc000481980, 0x2b99cc0, 0xc00005c488, 0x0, 0x0)
/__w/cortex/cortex/pkg/util/services/manager.go:145 +0xbb
github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run(0xc000ba9500, 0xc000b25d40, 0x4)
/__w/cortex/cortex/pkg/cortex/cortex.go:416 +0x91c
main.main()
/__w/cortex/cortex/cmd/cortex/main.go:189 +0xe3b
goroutine 4513530 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0e90c8780, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc18aab57a0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc078bd3200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4578956 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc079945980)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4578953 [IO wait]:
internal/poll.runtime_pollWait(0x7f599ab30a80, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00eda7c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00eda7c80, 0xc1ace34000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00eda7c80, 0xc1ace34000, 0x8000, 0x8000, 0x10401, 0x0, 0x1040102ade864)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0059b2508, 0xc1ace34000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1ea500f00, 0xc09ba61a78, 0x9, 0x9, 0xc001c90700, 0x7f63e74250f8, 0xc00044a780)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1ea500f00, 0xc09ba61a78, 0x9, 0x9, 0x9, 0xa6fc25, 0xc1931cba40, 0xc1931c0004)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc09ba61a78, 0x9, 0x9, 0x2b41140, 0xc1ea500f00, 0x0, 0x0, 0xc1931cba40, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc09ba61a40, 0xc1931cba40, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc079945680, 0xc19a278a80, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc079945680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc079945680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 307 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc00048c7c0)
/__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
/__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80
goroutine 92 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0007d4c00)
/__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:276 +0x100
created by go.opencensus.io/stats/view.init.0
/__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:34 +0x68
goroutine 161 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/alertmanager.init.0.func1()
/__w/cortex/cortex/pkg/alertmanager/alertmanager.go:95 +0x55
created by github.com/cortexproject/cortex/pkg/alertmanager.init.0
/__w/cortex/cortex/pkg/alertmanager/alertmanager.go:91 +0x35
goroutine 268 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0006c7100)
/__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
/__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80
goroutine 304 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc00048c500)
/__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
/__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80
goroutine 471 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549
goroutine 386 [select]:
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollControllerWithTicker(0xc0005c0a90, 0xc00045c730)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:144 +0xd0
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollController(0xc0005c0a90)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:139 +0x6e
created by github.com/uber/jaeger-client-go.NewRemotelyControlledSampler
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:86 +0x119
goroutine 378 [select]:
github.com/uber/jaeger-client-go/utils.(*reconnectingUDPConn).reconnectLoop(0xc0005b7110, 0x6fc23ac00)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:70 +0xed
created by github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x10f
goroutine 379 [select]:
github.com/uber/jaeger-client-go.(*remoteReporter).processQueue(0xc000b1fb00)
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:296 +0x120
created by github.com/uber/jaeger-client-go.NewRemoteReporter
/__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x1a0
goroutine 472 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 473 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 474 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 475 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 476 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 477 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 478 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 479 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 480 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 481 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 482 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 483 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 484 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 485 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 486 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 487 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 488 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 489 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 490 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 491 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 492 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 493 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 494 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 495 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 496 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 497 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 498 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 499 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 500 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 501 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 502 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 503 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 504 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 505 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 506 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 507 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 508 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 509 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 510 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 511 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 512 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 513 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 514 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 515 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 516 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 517 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 518 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 519 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 520 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 521 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da780)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 580 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549
goroutine 584 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 585 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 581 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 583 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 582 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 586 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 587 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 588 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 589 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 590 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 591 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 592 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 609 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 610 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 611 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 612 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 613 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 614 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 615 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 616 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 617 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 618 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 619 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 620 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 621 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 622 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 623 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 624 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 625 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 626 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 627 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 628 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 629 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 630 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 631 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 632 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 633 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 634 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 635 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 636 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 637 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 638 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 639 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 640 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 641 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 642 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 643 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 644 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 645 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 646 [select, 5 minutes]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0009da870)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 743 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549
goroutine 745 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 744 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 746 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 747 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 748 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 749 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 750 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 751 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 752 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 753 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 754 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 755 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 756 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 757 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 758 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 759 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 760 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 761 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 762 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 763 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 764 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 765 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 766 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 767 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 768 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 769 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 770 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 771 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 772 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 773 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 774 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 775 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 776 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 777 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 778 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 779 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 780 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 781 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 782 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 783 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 784 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 785 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 786 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 787 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 788 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 789 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 790 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 791 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 792 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 793 [select]:
github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc000fa6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102
created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594
goroutine 222 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc0004819e0, 0x2ba22c0, 0xc0001a5140)
/__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c
goroutine 223 [chan receive, 23 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000481a40, 0x2ba22c0, 0xc0001a51c0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c
goroutine 224 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000481aa0, 0x2ba22c0, 0xc0001a5260)
/__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c
goroutine 801 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000481b00, 0x2ba22c0, 0xc0001a52c0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c
goroutine 802 [chan receive, 23 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc000481bc0, 0x2b7fac0, 0xc0001a5380)
/__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener
/__w/cortex/cortex/pkg/util/services/manager.go:243 +0xfb
goroutine 803 [select, 24 minutes]:
github.com/weaveworks/common/signals.(*Handler).Loop(0xc000845290)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/signals/signals.go:48 +0x1d7
github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run.func4(0xc000845290, 0xc000481980)
/__w/cortex/cortex/pkg/cortex/cortex.go:405 +0x2b
created by github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run
/__w/cortex/cortex/pkg/cortex/cortex.go:404 +0x71b
goroutine 804 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006d0870, 0x2b99c80, 0xc000216c00, 0x4, 0xc0009dcae0, 0xc000e81001, 0xc000898e88)
/__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006d0870, 0x2b99c80, 0xc000216c00, 0xc0004819e0, 0x26b3980)
/__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000bf8000, 0x2b99c80, 0xc000216c00, 0x26b85c8, 0xc0006d0918)
/__w/cortex/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0900)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 805 [select, 23 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000a0d440, 0x2b99c80, 0xc000216c80, 0x4, 0xc00017c540, 0x1, 0xc000893e88)
/__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000a0d440, 0x2b99c80, 0xc000216c80, 0xc000481a40, 0x26b3980)
/__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000216ac0, 0x2b99c80, 0xc000216c80, 0x26b85c8, 0xc000a0d4e8)
/__w/cortex/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000a0d4d0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 806 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006d05a0, 0x2b99c80, 0xc000216d80, 0x4, 0xc0009dc6c0, 0xc000e81001, 0xc0009b3e88)
/__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006d05a0, 0x2b99c80, 0xc000216d80, 0xc000481aa0, 0x26b3980)
/__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000b25e00, 0x2b99c80, 0xc000216d80, 0x26b85c8, 0xc0006d0648)
/__w/cortex/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0630)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 807 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006d06c0, 0x2b99c80, 0xc000216e00, 0x4, 0xc0009dc960, 0xc000481b00, 0x26b3980)
/__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006d06c0, 0x2b99c80, 0xc000216e00, 0x0, 0xc0006d0750)
/__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000b25f80, 0x2b99c80, 0xc000216e00, 0x26b85c8, 0xc0006d0768)
/__w/cortex/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0750)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 808 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1(0x2b99c80, 0xc000217e00, 0x2, 0xc00103c768)
/__w/cortex/cortex/pkg/cortex/server_service.go:28 +0xfb
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d06c0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 809 [chan receive, 24 minutes]:
github.com/weaveworks/common/server.(*Server).Run(0xc000b262c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:414 +0x15b
github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1.1(0xc000bb0060, 0xc000b262c0)
/__w/cortex/cortex/pkg/cortex/server_service.go:25 +0x57
created by github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1
/__w/cortex/cortex/pkg/cortex/server_service.go:23 +0x5e
goroutine 810 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/cortex.ignoreSignalHandler.Loop(0xc0009dc7e0)
/__w/cortex/cortex/pkg/cortex/server_service.go:65 +0x34
github.com/weaveworks/common/server.(*Server).Run.func1(0xc000b262c0, 0xc000481d40)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:374 +0x3a
created by github.com/weaveworks/common/server.(*Server).Run
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:373 +0x6b
goroutine 811 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c073dd58, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0002f2398, 0x72, 0x0, 0x0, 0x253657d)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0002f2380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4
net.(*netFD).accept(0xc0002f2380, 0xe853b8885b027f48, 0x0, 0xe853b8885b027f48)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00080fc20, 0x601742e8, 0xc000a00e08, 0x4db7f6)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00080fc20, 0xc000a00e58, 0x18, 0xc000e81980, 0x8156cc)
/usr/local/go/src/net/tcpsock.go:261 +0x64
net/http.(*Server).Serve(0xc000bf6000, 0x2b87e80, 0xc00080fc20, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2930 +0x25d
github.com/weaveworks/common/server.(*Server).Run.func2(0xc000b262c0, 0xc000481d40)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:384 +0x126
created by github.com/weaveworks/common/server.(*Server).Run
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:381 +0x97
goroutine 812 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c073dc78, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0002f2518, 0x72, 0x0, 0x0, 0x253657d)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0002f2500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4
net.(*netFD).accept(0xc0002f2500, 0x37e11d600, 0x0, 0x0)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00080fc40, 0xc001042e50, 0xc001042e58, 0x18)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00080fc40, 0x26b6670, 0xc0002c3380, 0x2bc4400, 0xc00a33bf90)
/usr/local/go/src/net/tcpsock.go:261 +0x64
google.golang.org/grpc.(*Server).Serve(0xc0002c3380, 0x2b87e80, 0xc00080fc40, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:621 +0x210
github.com/weaveworks/common/server.(*Server).Run.func3(0xc000b262c0, 0xc000481d40)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:403 +0x4e
created by github.com/weaveworks/common/server.(*Server).Run
/__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:402 +0x13b
goroutine 800 [select]:
github.com/cortexproject/cortex/pkg/util/runtimeconfig.(*Manager).loop(0xc0006d07e0, 0x2b99c80, 0xc0008c0240, 0x0, 0x0)
/__w/cortex/cortex/pkg/util/runtimeconfig/manager.go:132 +0x13f
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0870)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 817 [syscall, 24 minutes]:
os/signal.signal_recv(0x26b85d0)
/usr/local/go/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.Notify.func1
/usr/local/go/src/os/signal/signal.go:127 +0x44
goroutine 596 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).running(0xc000b1fc80, 0x2b99c80, 0xc000bf9980, 0x26b85c8, 0xc0006d05b8)
/__w/cortex/cortex/pkg/ring/kv/memberlist/kv_init_service.go:52 +0xd6
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d05a0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 813 [semacquire, 5 minutes]:
sync.runtime_Semacquire(0xc1fc5dbfa8)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1fc5dbfa0)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks(0xc0001ce000, 0x2b99c80, 0xc000e9e080, 0x26b38d8, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:225 +0x53e
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).SyncBlocks(0xc0001ce000, 0x2b99c80, 0xc000e9e080, 0x1, 0x2bb6b40)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:138 +0x4b
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).syncStores(0xc000863100, 0x2b99c80, 0xc000e9e080, 0x2538ef7, 0x8)
/__w/cortex/cortex/pkg/storegateway/gateway.go:312 +0x23e
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).running(0xc000863100, 0x2b99c80, 0xc000e9e080, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/gateway.go:283 +0x47f
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000a0d440)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 814 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc00019c9c0, 0x2ba22c0, 0xc000c0c060)
/__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c
goroutine 815 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc00019ca80, 0x2ba22c0, 0xc000c0c0a0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
/__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c
goroutine 816 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc00019ccc0, 0x2b7fac0, 0xc000c0c0e0)
/__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener
/__w/cortex/cortex/pkg/util/services/manager.go:243 +0xfb
goroutine 833 [select]:
github.com/cortexproject/cortex/pkg/ring.(*BasicLifecycler).running(0xc000a0bee0, 0x2b99c80, 0xc000e9e0c0, 0x0, 0x0)
/__w/cortex/cortex/pkg/ring/basic_lifecycler.go:189 +0x18a
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0ea0)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 834 [select]:
net/http.(*persistConn).roundTrip(0xc27abf85a0, 0xc00d218180, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2559 +0x763
net/http.(*Transport).roundTrip(0xc000ce5680, 0xc238509200, 0xf8, 0xc000e9d101, 0xc238509200)
/usr/local/go/src/net/http/transport.go:582 +0xa88
net/http.(*Transport).RoundTrip(0xc000ce5680, 0xc238509200, 0xc000ce5680, 0xbffdeea00b8fc4bf, 0x1614ea167c3)
/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc238509100, 0x2b485c0, 0xc000ce5680, 0xbffdeea00b8fc4bf, 0x1614ea167c3, 0x3f6bae0, 0xc00357c648, 0xbffdeea00b8fc4bf, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:252 +0x43e
net/http.(*Client).send(0xc000bfbe60, 0xc238509100, 0xbffdeea00b8fc4bf, 0x1614ea167c3, 0x3f6bae0, 0xc00357c648, 0x0, 0x1, 0xc2027aa000)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).do(0xc000bfbe60, 0xc238509100, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:699 +0x44a
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:567
github.com/hashicorp/consul/api.(*Client).doRequest(0xc000bfc6c0, 0xc13edbcfc0, 0xc000e9d638, 0xc1f5b45800, 0x14, 0xc13edbcfc0)
/__w/cortex/cortex/vendor/github.com/hashicorp/consul/api/api.go:875 +0xb8
github.com/hashicorp/consul/api.(*KV).getInternal(0xc0001a2a08, 0x2542ee3, 0xd, 0x0, 0xc147d489a0, 0x7f63e741af10, 0xc00122e600, 0x216b600, 0x3f99388)
/__w/cortex/cortex/vendor/github.com/hashicorp/consul/api/kv.go:131 +0x285
github.com/hashicorp/consul/api.(*KV).Get(0xc0001a2a08, 0x2542ee3, 0xd, 0xc147d489a0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/hashicorp/consul/api/kv.go:65 +0xa0
github.com/cortexproject/cortex/pkg/ring/kv/consul.consulMetrics.Get.func1(0x2b99d40, 0xc1ea93b530, 0x3, 0xbffdee9b0b8f5a49)
/__w/cortex/cortex/pkg/ring/kv/consul/metrics.go:44 +0x126
github.com/weaveworks/common/instrument.CollectedRequest(0x2b99d40, 0xc0006780f0, 0x25309b1, 0x3, 0x2b846c0, 0xc000486420, 0x26b5d20, 0xc000e9d980, 0xb0, 0xa0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:152 +0x269
github.com/cortexproject/cortex/pkg/ring/kv/consul.consulMetrics.Get(0x2ba2480, 0xc0001a2a08, 0x2542ee3, 0xd, 0xc147d489a0, 0xc000e9db60, 0xa8, 0x2adee68, 0xc00007ddd8)
/__w/cortex/cortex/pkg/ring/kv/consul/metrics.go:41 +0x144
github.com/cortexproject/cortex/pkg/ring/kv/consul.(*Client).WatchKey(0xc0003c8d90, 0x2b99d40, 0xc0006780f0, 0x2542ee3, 0xd, 0xc000cb0090)
/__w/cortex/cortex/pkg/ring/kv/consul/client.go:224 +0x4ff
github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey.func1(0x2b99d40, 0xc0006780f0, 0x8, 0xbffded25f440deda)
/__w/cortex/cortex/pkg/ring/kv/metrics.go:91 +0x5e
github.com/weaveworks/common/instrument.CollectedRequest(0x2b99c80, 0xc000e9e100, 0x25384df, 0x8, 0x2b846c0, 0xc0001a2a20, 0x26b5d20, 0xc000e9ddf0, 0x0, 0x10)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:152 +0x269
github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey(0x2bb65a0, 0xc0003c8d90, 0xc0001a2a20, 0x2b99c80, 0xc000e9e100, 0x2542ee3, 0xd, 0xc000cb0090)
/__w/cortex/cortex/pkg/ring/kv/metrics.go:90 +0x101
github.com/cortexproject/cortex/pkg/ring.(*Ring).loop(0xc0006ead80, 0x2b99c80, 0xc000e9e100, 0x26b85c8, 0xc0006d0f48)
/__w/cortex/cortex/pkg/ring/ring.go:235 +0x95
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006d0f30)
/__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
/__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8
goroutine 236067 [semacquire]:
sync.runtime_Semacquire(0xc0182a84c8)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0182a84c0)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc028cbc480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc028cbc480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 884 [semacquire]:
sync.runtime_SemacquireMutex(0xc007cd033c, 0xc0f31dbd00, 0x1)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*Mutex).lockSlow(0xc007cd0338)
/usr/local/go/src/sync/mutex.go:138 +0xfc
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0xc007cd0338)
/usr/local/go/src/sync/rwmutex.go:98 +0x97
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).unload(0xc007cd02c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:254 +0xd5
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).closeIdleReaders(0xc000b25680)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:109 +0x86
github.com/thanos-io/thanos/pkg/block/indexheader.NewReaderPool.func1(0xc000b25680, 0x1bf08eb000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:58 +0x42
created by github.com/thanos-io/thanos/pkg/block/indexheader.NewReaderPool
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:52 +0x13e
goroutine 4514542 [IO wait]:
internal/poll.runtime_pollWait(0x7f5886b2e7a8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e598, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e580, 0xc1fe14c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e580, 0xc1fe14c000, 0x8000, 0x8000, 0x20305b, 0x420bb5, 0xc16fdff5a0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d460d0, 0xc1fe14c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1cb011f20, 0xc1fafa3458, 0x9, 0x9, 0xc200ac2000, 0x1007f63e740a7d0, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1cb011f20, 0xc1fafa3458, 0x9, 0x9, 0x9, 0x601742e6, 0xc079ec9dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1fafa3458, 0x9, 0x9, 0x2b41140, 0xc1cb011f20, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1fafa3420, 0xc265a9a5e0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc054d93b00, 0xc23cc7b020, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc054d93b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc054d93b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582128 [select]:
net/http.(*persistConn).readLoop(0xc0f78e2240)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4513603 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc175158d70, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc190600300, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc10a44af00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 209257 [semacquire]:
sync.runtime_Semacquire(0xc0b41fb338)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0b41fb330)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0e542a480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0e542a480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4513531 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc078bd3200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582747 [select]:
net/http.(*persistConn).writeLoop(0xc0fc7806c0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4511255 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc047d69360, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d7db40c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1245db680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4514787 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8c80, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d7bc0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc52a933c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4578948 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0f698b040, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ea500c00, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1e65f3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 98114 [semacquire]:
sync.runtime_Semacquire(0xc019c06468)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc019c06460)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc00f65e300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc00f65e300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514703 [IO wait]:
internal/poll.runtime_pollWait(0x7f59df09f240, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281a00, 0xc233518000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281a00, 0xc233518000, 0x8000, 0x8000, 0x203066, 0x420bb5, 0xc19b9a1400)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa430, 0xc233518000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc2442d6420, 0xc0460c2f18, 0x9, 0x9, 0x463b40, 0xc0460c2f08, 0xc227d32dc0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc2442d6420, 0xc0460c2f18, 0x9, 0x9, 0x9, 0x56, 0x18, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c2f18, 0x9, 0x9, 0x2b41140, 0xc2442d6420, 0x0, 0x0, 0x0, 0xc02d3b8870)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2ee0, 0xc25836ae50, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477c80, 0xc22f1daff0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4516941 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003a9e900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4508841 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1ae679630, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d7d860c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc4b81e0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4517379 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a90c96ca8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1c30b4398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1c30b4380, 0xc106310000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1c30b4380, 0xc106310000, 0x8000, 0x8000, 0x800010601, 0x0, 0x10601000000e1)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1f017c028, 0xc106310000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1a7997560, 0xc0dd42b378, 0x9, 0x9, 0xc001b1fd40, 0x457243, 0xb5ba6808de07)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1a7997560, 0xc0dd42b378, 0x9, 0x9, 0x9, 0xbffdee9b0a9faceb, 0x15ca5998803, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0dd42b378, 0x9, 0x9, 0x2b41140, 0xc1a7997560, 0x0, 0x0, 0xc255227c20, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0dd42b340, 0xc255227c20, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1038bcf00, 0xc2039253e0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4474331 [semacquire]:
sync.runtime_Semacquire(0xc217e2e2b0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc217e2e2a8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc217e2e2a0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1dae62420)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1ed00b7d0, 0x255e040, 0x18, 0xc1ef8073f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1ea4afe60, 0x2bbccc0, 0xc18059e8e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1ea4afe60, 0x7f63bf19f960, 0xc2150fc440, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1ea4afe60, 0x2bbbd60, 0xc2150fc440, 0xc000863100, 0xc0092a3688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc18059e840, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18059e820, 0xc18059e6c0, 0x26b3908, 0xc18059e820, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18059e820, 0x2b99d40, 0xc1ed00b680)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18059e800, 0xc18059e6c0, 0xc18059e6e0, 0x7f63e743b5d8, 0xc006081c80)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18059e800, 0xc18059e6c0, 0xc18059e6e0, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18059e800, 0x2b99d40, 0xc1ed00b650)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0xc18059e700, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0x1, 0xc18059e680)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0xc18059e720, 0x2ac7d009, 0xb59d31f3ee84)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0x203060, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0xc18059e740, 0xc18059e6c0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc0f099ac68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d73e9c80, 0xc18059e6c0, 0x26b3908, 0x2b99d40, 0xc1ed00b5f0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876100, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876100, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0182a84c0, 0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876100)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 95630 [semacquire]:
sync.runtime_Semacquire(0xc075a62c58)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc075a62c50)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc071fc8780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc071fc8780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4516942 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d8f043b8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281e18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281e00, 0xc142f18000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281e00, 0xc142f18000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000072)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa480, 0xc142f18000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1f71bab40, 0xc0a65ff458, 0x9, 0x9, 0xc079ecdd40, 0x457243, 0xb5ba67ff0b46)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1f71bab40, 0xc0a65ff458, 0x9, 0x9, 0x9, 0xbffdee9b0a94b6f1, 0x15ca58e922e, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0a65ff458, 0x9, 0x9, 0x2b41140, 0xc1f71bab40, 0x0, 0x0, 0xc253ccee40, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a65ff420, 0xc253ccee40, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003a9e900, 0xc22c875980, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc003a9e900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc003a9e900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4577950 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0fe25c1e0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ea4a8420, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0c41d3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582750 [select]:
net/http.(*persistConn).readLoop(0xc10609c000)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4084494 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc17b38de00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0xc1efbc95c0, 0x40cfa8, 0x60, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77
github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9
github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x0, 0x8000101, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b
github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc258f73f20, 0x25, 0x17a4f206, 0x1cfc932, 0x0, 0x7f59e0599060, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a
github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc253cca260, 0x20, 0x17a4f206, 0x1cfc932, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc43967fb40, 0x2b44ea0, 0xc19f5801e0, 0xc212664000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xad5fde977f6f7601, 0x5e8f6986b1914cbd, 0xc1bdd68140, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xad5fde977f6f7601, 0x5e8f6986b1914cbd, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xad5fde977f6f7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0034b7b00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4515975 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0de578000)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4561337 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc08f604c80, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc21d46e060, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc136f0c900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4566211 [semacquire]:
sync.runtime_Semacquire(0xc1e07b48b0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1e07b48a8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1e07b48a0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1e07b4a50)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0e785d680, 0x255e040, 0x18, 0xc0fe19d3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1a83f3140, 0x2bbccc0, 0xc0d2c544a0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1a83f3140, 0x7f63bf19f960, 0xc1def819d0, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1a83f3140, 0x2bbbd60, 0xc1def819d0, 0xc000863100, 0xc0fe19d688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0d2c54400, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0d2c543e0, 0xc0d2c54280, 0x26b3908, 0xc0d2c543e0, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0d2c543e0, 0x2b99d40, 0xc0e785d530)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0d2c543c0, 0xc0d2c54280, 0xc0d2c542a0, 0x7f63e743ca30, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0d2c543c0, 0xc0d2c54280, 0xc0d2c542a0, 0x2b99d01, 0xc0d2c543c0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0d2c543c0, 0x2b99d40, 0xc0e785d500)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0xc0d2c542c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0x1, 0xc0d2c54240)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0xc0d2c542e0, 0x2bb9afc2, 0xb5b83509563d)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0x203034, 0x8701ba)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0xc0d2c54300, 0xc0d2c54280, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0005e6c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ac051b00, 0xc0d2c54280, 0x26b3908, 0x2b99d40, 0xc0e785d4a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780, 0xc278c53e00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780, 0xc278c53e00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0f7c82930, 0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780, 0xc278c53e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 336244 [semacquire]:
sync.runtime_Semacquire(0xc0051527a8)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0051527a0)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0294cb380)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0294cb380)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 211498 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc0ca383b48)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0ca383b40)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514704 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8a50, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d76e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582663 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d97cf9f8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1a0eadf18, 0x72, 0x9800, 0x98a0, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1a0eadf00, 0xc23f932000, 0x98a0, 0x98a0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1a0eadf00, 0xc23f932000, 0x98a0, 0x98a0, 0x203092, 0x420bb5, 0xc2495d8960)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc007b7ef08, 0xc23f932000, 0x98a0, 0x98a0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc2495d8960, 0xc23f932000, 0x98a0, 0x98a0, 0x3448, 0x9893, 0xc01ac02e90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc073195058, 0x2b41380, 0xc2495d8960, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc073194e00, 0x2b48520, 0xc007b7ef08, 0x5, 0xc007b7ef08, 0x3438)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc073194e00, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc073194e00, 0xc227dabab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0fa6fe480, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc23e49a480, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc1eb15c240, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc003233300, 0xc227dabab2, 0x254e, 0x254e, 0x3438, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc003233300, 0xc227dabab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc003233340, 0xc227dabab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc003233340, 0xc227da6000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc029bd1380, 0xc029bd13e0, 0xc029bd1440, 0xc00926e860, 0xc225a725d0, 0xc044daa000, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4556672 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c00f9be0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc008d7ed18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc008d7ed00, 0xc145b80000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc008d7ed00, 0xc145b80000, 0x8000, 0x8000, 0x20312f, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc005f6a6b8, 0xc145b80000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1ce759500, 0xc07662c038, 0x9, 0x9, 0xc005f19100, 0x1007f63e740a108, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1ce759500, 0xc07662c038, 0x9, 0x9, 0x9, 0x601742e3, 0xc047ef7dc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc07662c038, 0x9, 0x9, 0x2b41140, 0xc1ce759500, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc07662c000, 0xc2504f92c0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc183057e00, 0xc2917e8bd0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc183057e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc183057e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4517377 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc08ae15220, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc100420660, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1038bcf00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582666 [select]:
net/http.(*persistConn).readLoop(0xc0b6675440)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4514796 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b92c0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0911800, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4515929 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00bc80d80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4557185 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1563d3130, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1a110a660, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc11159f080)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4486232 [IO wait]:
internal/poll.runtime_pollWait(0x7f59decac3e0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc22b860118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc22b860100, 0xc1b5e4a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc22b860100, 0xc1b5e4a000, 0x8000, 0x8000, 0x400000801, 0xc70000000f, 0x8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004ffb038, 0xc1b5e4a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc12843ae40, 0xc1df964e38, 0x9, 0x9, 0xc1dd153ce0, 0xc13c543df7, 0xc13c543d80)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc12843ae40, 0xc1df964e38, 0x9, 0x9, 0x9, 0x406594, 0xc0b771eae0, 0xc13c543df7)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1df964e38, 0x9, 0x9, 0x2b41140, 0xc12843ae40, 0x0, 0xc100000000, 0x2124f40, 0xc75045ca70)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1df964e00, 0xc75045ca60, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1c1c82180, 0xc122530ea0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1c1c82180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1c1c82180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4508536 [select]:
net/http.(*persistConn).writeLoop(0xc0deef0a20)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4568631 [semacquire]:
sync.runtime_Semacquire(0xc0e77e0be0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0e77e0bd8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc0e77e0bd0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0e5e87f80)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0ffa08ab0, 0x255e040, 0x18, 0xc024c653f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc065e39800, 0x2bbccc0, 0xc1685ff740, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc065e39800, 0x7f63bf19f960, 0xc17d28c520, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc065e39800, 0x2bbbd60, 0xc17d28c520, 0xc000863100, 0xc024c65688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc07f618a80, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc07f618a60, 0xc07f618900, 0x26b3908, 0xc07f618a60, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc07f618a60, 0x2b99d40, 0xc1f17b2000)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc07f618a40, 0xc07f618900, 0xc07f618920, 0x7f63e742ef10, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc07f618a40, 0xc07f618900, 0xc07f618920, 0x2b99d01, 0xc07f618a40)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc07f618a40, 0x2b99d40, 0xc47d55edb0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0xc07f618940, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0x1, 0xc07f6188c0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0xc07f618960, 0x3b244204, 0xb5b84473e87a)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0x20301f, 0x8701ba)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0xc07f618980, 0xc07f618900, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc12b912c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0adb61200, 0xc07f618900, 0x26b3908, 0x2b99d40, 0xc47d55ed50)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0de578000, 0xc0ed67af00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0de578000, 0xc0ed67af00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc1296eeaf0, 0xc0002c3380, 0x2bc6d40, 0xc0de578000, 0xc0ed67af00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4582807 [select]:
net/http.(*persistConn).roundTrip(0xc0fd6c8000, 0xc00473c480, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2559 +0x763
net/http.(*Transport).roundTrip(0xc000ce5540, 0xc0f6b2e200, 0xc11fd686c0, 0x40f967, 0xc0000a7bf8)
/usr/local/go/src/net/http/transport.go:582 +0xa88
net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc0f6b2e200, 0xc000ce5540, 0x0, 0x0)
/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc0f6b2e200, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xca27c505f0, 0x2537f0f, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:252 +0x43e
net/http.(*Client).send(0xc000bfaba0, 0xc0f6b2e200, 0x0, 0x0, 0x0, 0xca27c505f0, 0x0, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).do(0xc000bfaba0, 0xc0f6b2e200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:699 +0x44a
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:567
github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50
github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409
github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc092731bc0, 0xc092731c20, 0xc092731ce0, 0xc1451f5130, 0xc23b892e70, 0xc143aa7d90, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4513606 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc175159e50, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ba7c7a40, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc10a44bc80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4084492 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc24ae98000, 0xc2270da000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e379920, 0xc2270da000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc23c574b40, 0xc2270da000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc0fe80cc80, 0x7f59e0599060, 0xc23c574b40, 0xc2270da000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc0fe80cc80, 0x7f59e0599060, 0xc23c574b40, 0xc2270da000, 0x8000, 0x8000, 0x2b6fc80, 0xc23c574b40, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0052ce3c0, 0x2b44ea0, 0xc0fe80cc80, 0xc2270da000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x1cc0a35fbeb97601, 0x161981381d9f8301, 0xc114951b00, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x1cc0a35fbeb97601, 0x161981381d9f8301, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x1cc0a35fbeb97601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0043ebe00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4084804 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc17b38dc80, 0xc20e372000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e379980, 0xc20e372000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc23c574c60, 0xc20e372000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc081243ef0, 0x7f59e0599060, 0xc23c574c60, 0xc20e372000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc081243ef0, 0x7f59e0599060, 0xc23c574c60, 0xc20e372000, 0x8000, 0x8000, 0x2b6fc80, 0xc23c574c60, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0026ad940, 0x2b44ea0, 0xc081243ef0, 0xc20e372000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x2856316f9d2d7701, 0xcee3ec3285783cc7, 0xc143f83080, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2856316f9d2d7701, 0xcee3ec3285783cc7, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2856316f9d2d7701, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc007568400, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4581870 [semacquire]:
sync.runtime_SemacquireMutex(0xc007cd0344, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc007cd02c0, 0xc0d278c2c8, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc1564447d0, 0xc114311bf0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc99c97fc00, 0xc07df928b0, 0x2, 0x2, 0x2d850e0a0cd5be5, 0x7854ca83079dac14, 0x61032ff49c49aef4, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0040c6990, 0xc99c97fc00, 0xc121ce5d40, 0xc07df928b0, 0x2, 0x2, 0xc121ce5920, 0x2b44f80, 0xc114311cb0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc21d477500, 0xc1297e9800)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc114311c80, 0xc150d53f10)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582835 [select]:
net/http.(*persistConn).writeLoop(0xc10297e000)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4574119 [semacquire]:
sync.runtime_SemacquireMutex(0xc00d68dc04, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00d68db80, 0xc03f1468b0, 0xb, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc1e89f57d0, 0xc1e07b4780, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc9a77a0f80, 0xc181398a80, 0x3, 0x3, 0xc255ef5390, 0xd, 0x1, 0xc1ed1fca80, 0x7f63e7423ca0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0077aba70, 0xc9a77a0f80, 0xc1d7d80c60, 0xc181398a80, 0x3, 0x3, 0xc1a83f3140, 0x2b44f80, 0xc1e07b48d0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x1, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e07b48a0, 0xc00915c690)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582748 [select]:
net/http.(*persistConn).readLoop(0xc0d0a00d80)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4513292 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc2a2000190, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1e6948060, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc065466f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 347184 [semacquire, 3 minutes]:
sync.runtime_Semacquire(0xc0eb34a3d8)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0eb34a3d0)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc09673ec00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc09673ec00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 97730 [semacquire]:
sync.runtime_Semacquire(0xc02c9b5d28)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc02c9b5d20)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0073fe780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0073fe780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514541 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc054d93b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4562284 [semacquire]:
sync.runtime_Semacquire(0xc1ea93a460)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1ea93a458)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1ea93a450, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1ea93a9c0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1ea93a390, 0x255e040, 0x18, 0xc249d033f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc13edbc1e0, 0x2bbccc0, 0xc1ea9303c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc13edbc1e0, 0x7f63bf19f960, 0xc000aa40f0, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc13edbc1e0, 0x2bbbd60, 0xc000aa40f0, 0xc000863100, 0xc101347688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1ea930320, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea930300, 0xc1ea9301a0, 0x26b3908, 0xc1ea930300, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea930300, 0x2b99d40, 0xc1ea93a240)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea9302e0, 0xc1ea9301a0, 0xc1ea9301c0, 0x7f63e741af10, 0xc06fbf1380)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea9302e0, 0xc1ea9301a0, 0xc1ea9301c0, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea9302e0, 0x2b99d40, 0xc1ea93a210)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0xc1ea9301e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0x1, 0xc1ea930160)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0xc1ea930200, 0xa94b9e9, 0xb5ba67f04466)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0x20307a, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0xc1ea930220, 0xc1ea9301a0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc05d8b0c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1da7b9bc0, 0xc1ea9301a0, 0x26b3908, 0x2b99d40, 0xc1ea93a1b0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc238508a00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc238508a00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc12c6a8860, 0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc238508a00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4577952 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d9b17ad0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc181fd1518, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc181fd1500, 0xc1ad662000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc181fd1500, 0xc1ad662000, 0x8000, 0x8000, 0x10401, 0x0, 0x1040102ade864)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150ab49b8, 0xc1ad662000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0bf00a360, 0xc0f27018b8, 0x9, 0x9, 0xc0061ee700, 0x7f63e742e180, 0xc06fbf1c80)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0bf00a360, 0xc0f27018b8, 0x9, 0x9, 0x9, 0xa6fc25, 0xc110468cf0, 0xc110460004)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0f27018b8, 0x9, 0x9, 0x2b41140, 0xc0bf00a360, 0x0, 0x0, 0xc110468cf0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f2701880, 0xc110468cf0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0c41d3800, 0xc1136af7a0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0c41d3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0c41d3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582833 [select]:
net/http.(*persistConn).writeLoop(0xc0f78e2240)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4532623 [semacquire]:
sync.runtime_Semacquire(0xc27a2de2e0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc27a2de2d8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc27a2de2d0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc27477a960)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc12dce4420, 0x255e040, 0x18, 0xc2394e93f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc13de3c660, 0x2bbccc0, 0xc06e14d200, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc13de3c660, 0x7f63bf19f960, 0xc1fa720ac0, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc13de3c660, 0x2bbbd60, 0xc1fa720ac0, 0xc000863100, 0xc2394e9688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc06e14d160, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc06e14d140, 0xc06e14cfe0, 0x26b3908, 0xc06e14d140, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc06e14d140, 0x2b99d40, 0xc12dce42d0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc06e14d120, 0xc06e14cfe0, 0xc06e14d000, 0x7f63e741b5d8, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc06e14d120, 0xc06e14cfe0, 0xc06e14d000, 0x2b99d01, 0xc06e14d120)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc06e14d120, 0x2b99d40, 0xc12dce42a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0xc06e14d020, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0x1, 0xc06e14cfa0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0xc06e14d040, 0xf1194f4, 0xb5b5c455576a)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0x20301b, 0x8701ba)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x7f59e185f400, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0xc06e14d060, 0xc06e14cfe0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc157146c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc07699ecc0, 0xc06e14cfe0, 0x26b3908, 0x2b99d40, 0xc12dce4240)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc12dde6b00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc12dde6b00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc126847560, 0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc12dde6b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4582523 [select]:
net/http.(*persistConn).writeLoop(0xc9aa80f7a0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4565087 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1ecc2c480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4557187 [IO wait]:
internal/poll.runtime_pollWait(0x7f63bf05f728, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e300, 0xc1de1bc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e300, 0xc1de1bc000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000006)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d460a0, 0xc1de1bc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc14ff80600, 0xc110dfcd58, 0x9, 0x9, 0xc071596d40, 0x457243, 0xb5b93b52c6bc)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc14ff80600, 0xc110dfcd58, 0x9, 0x9, 0x9, 0xbffdee99c7eec8e9, 0x15b78e2b1ff, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc110dfcd58, 0x9, 0x9, 0x2b41140, 0xc14ff80600, 0x0, 0x0, 0xc25c7680c0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc110dfcd20, 0xc25c7680c0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc11159f080, 0xc27bf06270, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc11159f080)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc11159f080)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4513608 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d8f04738, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00cfa2718, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00cfa2700, 0xc1ffa5e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00cfa2700, 0xc1ffa5e000, 0x8000, 0x8000, 0x20308e, 0x420bb5, 0xc23a5d9d80)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc9e49f40f8, 0xc1ffa5e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc10cfe4fc0, 0xc1b5e6c658, 0x9, 0x9, 0xc00596d180, 0x1007f63e742a848, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc10cfe4fc0, 0xc1b5e6c658, 0x9, 0x9, 0x9, 0x601742dd, 0xc003bfddc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1b5e6c658, 0x9, 0x9, 0x2b41140, 0xc10cfe4fc0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1b5e6c620, 0xc0168a5f80, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc10a44bc80, 0xc24500a720, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc10a44bc80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc10a44bc80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4511072 [IO wait]:
internal/poll.runtime_pollWait(0x7f58819b36b0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc16897a498, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc16897a480, 0xc249901000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc16897a480, 0xc249901000, 0x1000, 0x1000, 0x4db33a, 0xb5bb70dfc9e3, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008464108, 0xc249901000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
net/http.(*connReader).Read(0xc2080077a0, 0xc249901000, 0x1000, 0x1000, 0xbffdeeb3102f39c8, 0x1730534d4cc, 0x3f6bae0)
/usr/local/go/src/net/http/server.go:797 +0xf4
bufio.(*Reader).fill(0xc1ab3ef920)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc1ab3ef920, 0x4, 0x1730534d4cc, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc245df2140, 0x2b99c80, 0xc1ebb5eac0)
/usr/local/go/src/net/http/server.go:1949 +0x9d7
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2962 +0x35c
goroutine 4514647 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9f7d390, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1c30b4218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1c30b4200, 0xc22f1dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1c30b4200, 0xc22f1dc000, 0x8000, 0x8000, 0x400000801, 0x1, 0x8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1f017c010, 0xc22f1dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc23b73ac60, 0xc0460c2c78, 0x9, 0x9, 0xc22f1dc034, 0x4, 0x18)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc23b73ac60, 0xc0460c2c78, 0x9, 0x9, 0x9, 0xc071597dc8, 0x40cfa8, 0x20)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c2c78, 0x9, 0x9, 0x2b41140, 0xc23b73ac60, 0x0, 0xc100000000, 0x2124f40, 0xc156f1e048)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2c40, 0xc156f1e050, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477200, 0xc22f1da510, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4464565 [running]:
goroutine running on other thread; stack unavailable
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4515976 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a90c95dc8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc23a27d418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc23a27d400, 0xc13642c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc23a27d400, 0xc13642c000, 0x8000, 0x8000, 0x800010601, 0x0, 0x106010000001a)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc52a84a000, 0xc13642c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d9dfc2a0, 0xc244dee1f8, 0x9, 0x9, 0xc0220bad40, 0x457243, 0xb5b93c5069f8)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d9dfc2a0, 0xc244dee1f8, 0x9, 0x9, 0x9, 0xbffdee99c8e997ee, 0x15b79dd8106, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc244dee1f8, 0x9, 0x9, 0x2b41140, 0xc1d9dfc2a0, 0x0, 0x0, 0xc260132340, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc244dee1c0, 0xc260132340, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0de578000, 0xc241efc4b0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0de578000)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0de578000)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 235963 [semacquire, 3 minutes]:
sync.runtime_Semacquire(0xc03f97d978)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc03f97d970)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 347451 [semacquire, 3 minutes]:
sync.runtime_Semacquire(0xc0b0cfd638)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0b0cfd630)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0158e2480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:755 +0xe9
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0158e2480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4510185 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0cae88cd0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1e950a960, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc19966bb00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4566154 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d9ceeb40, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1acbcfd40, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0ab132a80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582746 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d9810148, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1e8c28498, 0x72, 0xe100, 0xe1c1, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1e8c28480, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1e8c28480, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x203092, 0x420bb5, 0xc24bf39da0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc029070e18, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc24bf39da0, 0xc25b8e0000, 0xe1c1, 0xe1c1, 0x0, 0x41310b, 0xc22856c900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc2011ad058, 0x2b41380, 0xc24bf39da0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc2011ace00, 0x2b48520, 0xc029070e18, 0x5, 0xc029070e18, 0x40c924)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc2011ace00, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc2011ace00, 0xc23dd31000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0fc7806c0, 0xc23dd31000, 0x1000, 0x1000, 0xc072dc6300, 0xc22856cc58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc23e84e180)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc23e84e180, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0ca347500)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0fc7806c0)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4511300 [IO wait]:
internal/poll.runtime_pollWait(0x7f59deb9dc60, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1a5d31d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1a5d31d80, 0xc1ff776000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1a5d31d80, 0xc1ff776000, 0x8000, 0x8000, 0x400000801, 0xc100000001, 0x8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150cbc078, 0xc1ff776000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d9d0fb60, 0xc0c3c41538, 0x9, 0x9, 0xc1ff776027, 0x4, 0x18)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d9d0fb60, 0xc0c3c41538, 0x9, 0x9, 0x9, 0xc00169ddc8, 0x40cfa8, 0x20)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0c3c41538, 0x9, 0x9, 0x2b41140, 0xc1d9d0fb60, 0x0, 0xc100000000, 0x2124f40, 0xc105e7fad8)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c3c41500, 0xc105e7faf0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc167bf0900, 0xc1d77a48d0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc167bf0900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc167bf0900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4501209 [semacquire]:
sync.runtime_SemacquireMutex(0xc00c4186b0, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc00c4186a8)
/usr/local/go/src/sync/rwmutex.go:103 +0x88
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00c418630, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c418630, 0xc1858568e1, 0x3, 0x7f4c11e8dc72, 0xe, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1ea70f500, 0xc06b51aa00, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1ea70f500, 0xc1a577d020, 0x2, 0x2, 0xc07905b480, 0xd, 0x1, 0xc005e16e00, 0x7f63e7424a30)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc004d1b1a0, 0xc1ea70f500, 0xc1d9cb25a0, 0xc1a577d020, 0x2, 0x2, 0xc182805b00, 0x2b44f80, 0xc245857890, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc17c65c740, 0xc236e088b0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc245857860, 0xc14393b960)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4522981 [IO wait]:
internal/poll.runtime_pollWait(0x7f5887153e30, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281d80, 0xc107cba000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281d80, 0xc107cba000, 0x8000, 0x8000, 0x203060, 0x420bb5, 0xc180257760)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa478, 0xc107cba000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc198be6de0, 0xc1c4a3ee38, 0x9, 0x9, 0xc005ea3180, 0x1007f63e741ca30, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc198be6de0, 0xc1c4a3ee38, 0x9, 0x9, 0x9, 0x601742e6, 0xc079ec8dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1c4a3ee38, 0x9, 0x9, 0x2b41140, 0xc198be6de0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1c4a3ee00, 0xc25ecfd460, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc4f1083800, 0xc00bde8cc0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc4f1083800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc4f1083800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514788 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc52a933c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4514916 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4565086 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0cdc1e780, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1359838c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1ecc2c480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4581469 [semacquire]:
sync.runtime_Semacquire(0xc1bc0bd7b0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1bc0bd7a8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1bc0bd7a0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1ae047bf0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1bc0bd6b0, 0x255e040, 0x18, 0xc15f1ab3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1893ba060, 0x2bbccc0, 0xc1e902a400, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1893ba060, 0x7f63bf19f960, 0xc00c028430, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1893ba060, 0x2bbbd60, 0xc00c028430, 0xc000863100, 0xc0ff3c1688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1e902a360, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e902a340, 0xc1e902a1e0, 0x26b3908, 0xc1e902a340, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e902a340, 0x2b99d40, 0xc1bc0bd560)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e902a320, 0xc1e902a1e0, 0xc1e902a200, 0x7f63e741fca0, 0xc0b6aae000)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e902a320, 0xc1e902a1e0, 0xc1e902a200, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e902a320, 0x2b99d40, 0xc1bc0bd530)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0xc1e902a220, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0x1, 0xc1e902a1a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0xc1e902a240, 0x15d5fd4b, 0xb5b9c06129c3)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0x20307a, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0xc1e902a260, 0xc1e902a1e0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc0005eac68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d21549c0, 0xc1e902a1e0, 0x26b3908, 0x2b99d40, 0xc1bc0bd4d0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc0f6f21b00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc0f6f21b00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc12c6a8860, 0xc0002c3380, 0x2bc6d40, 0xc003a9e900, 0xc0f6f21b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4514800 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1503a02d0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0911b60, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430c00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4516666 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0b393b4a0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc14c6d0d80, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc2fd38d200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4514700 [IO wait]:
internal/poll.runtime_pollWait(0x7f5885067988, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e280, 0xc233500000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e280, 0xc233500000, 0x8000, 0x8000, 0x20308b, 0x420bb5, 0xc22c886400)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d46098, 0xc233500000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc23b73bf80, 0xc0460c2e38, 0x9, 0x9, 0xc006200700, 0x1007f63e7426180, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc23b73bf80, 0xc0460c2e38, 0x9, 0x9, 0x9, 0x601742dd, 0xc0594ebdc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c2e38, 0x9, 0x9, 0x2b41140, 0xc23b73bf80, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2e00, 0xc16d8efe40, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477b00, 0xc22f1dade0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514794 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4515946 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9d0d468, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1c30b4498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1c30b4480, 0xc2023f2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1c30b4480, 0xc2023f2000, 0x8000, 0x8000, 0x203039, 0x420bb5, 0xc0e721c2a0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1f017c038, 0xc2023f2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc10cfe4f60, 0xc04a34eb98, 0x9, 0x9, 0xc1d4e89c00, 0x1007f63e74290f8, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc10cfe4f60, 0xc04a34eb98, 0x9, 0x9, 0x9, 0x601742e6, 0xc038e1cdc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc04a34eb98, 0x9, 0x9, 0x2b41140, 0xc10cfe4f60, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc04a34eb60, 0xc26727f5e0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc028eec180, 0xc2459a0630, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc028eec180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc028eec180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514791 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc52a933e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4513607 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc10a44bc80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582153 [semacquire]:
sync.runtime_SemacquireMutex(0xc00ddea134, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).IndexVersion(0xc00ddea0b0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:141 +0x1f7
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0049f9300, 0xc084f17b90, 0x2, 0x2, 0x55d6ca13f4bfe0e2, 0x23aeb4e8816c5b14, 0x6a35026af49e6978, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1639 +0xfbf
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00618cb10, 0xc0049f9300, 0xc14a10a120, 0xc084f17b90, 0x2, 0x2, 0xc197b33380, 0x2b44f80, 0xc1e8866780, 0xc00017c0b8, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e8866750, 0xc1e8b404d0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4515935 [IO wait]:
internal/poll.runtime_pollWait(0x7f59decad100, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1528d3698, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1528d3680, 0xc2473f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1528d3680, 0xc2473f4000, 0x8000, 0x8000, 0x203059, 0x420bb5, 0xc167ccaea0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1cb614160, 0xc2473f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc19149f860, 0xc1df964578, 0x9, 0x9, 0xc00d1b5138, 0xc000894d60, 0x18)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc19149f860, 0xc1df964578, 0x9, 0x9, 0x9, 0xc000894dc8, 0x40cfa8, 0xc07661ae70)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1df964578, 0x9, 0x9, 0x2b41140, 0xc19149f860, 0x0, 0x0, 0x0, 0xc07661ae60)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1df964540, 0xc1f3148c80, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00bc80f00, 0xc23ec67a10, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc00bc80f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc00bc80f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4516667 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc2fd38d200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582737 [runnable]:
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0ca346720, 0xc0ca346780, 0xc0ca346840, 0xc00d118950, 0xc1e8b46ea0, 0xc142ad7a20, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4582274 [IO wait]:
internal/poll.runtime_pollWait(0x7f599ab310a0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc19c7d8b98, 0x72, 0xdd00, 0xddd6, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc19c7d8b80, 0xc241cbe000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc19c7d8b80, 0xc241cbe000, 0xddd6, 0xddd6, 0x203094, 0x420bb5, 0xc253c642e0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150304b98, 0xc241cbe000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc253c642e0, 0xc241cbe000, 0xddd6, 0xddd6, 0x1200, 0xddc9, 0xc0aad98e90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07ab77058, 0x2b41380, 0xc253c642e0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07ab76e00, 0x2b48520, 0xc150304b98, 0x5, 0xc150304b98, 0x11f0)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07ab76e00, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07ab76e00, 0xc224287e42, 0x61be, 0x61be, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc24422a7e0, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc23e379680, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc23b13d340, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc00702a680, 0xc224287e42, 0x61be, 0x61be, 0x11f0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc00702a680, 0xc224287e42, 0x61be, 0x61be, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc00702a6c0, 0xc224287e42, 0x61be, 0x61be, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc00702a6c0, 0xc224286000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc26a56a240, 0xc26a56a2a0, 0xc26a56a300, 0xc000aa4e50, 0xc1ea93b710, 0xc147d48b00, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4577212 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc52a6fd200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4514645 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc022deff40, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc23b73b6e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4510177 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1a79dd180, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc22a263da0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc19966b800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4388238 [running]:
goroutine running on other thread; stack unavailable
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4514691 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477500)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4502851 [semacquire]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc14319a860, 0x8, 0xc1507c8780, 0x28, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1e6c36a00, 0xc0f27e6c40, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1e6c36a00, 0xc1c746a818, 0x1, 0x1, 0x28bcc65066b49d01, 0x30077b56078f8525, 0x3e90ff60c125ea02, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc1e6c36a00, 0xc14a332c00, 0xc1c746a818, 0x1, 0x1, 0xc088e9d200, 0x2b44f80, 0xc036d84ff0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc165568ee0, 0xc17c325dd0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc036d84fc0, 0xc00d5bed20)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4464215 [running]:
goroutine running on other thread; stack unavailable
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582839 [select]:
net/http.(*persistConn).writeLoop(0xc0f7ded7a0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4514792 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9c50a88, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281b00, 0xc233560000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281b00, 0xc233560000, 0x8000, 0x8000, 0x4647d0, 0xc1734a2300, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa450, 0xc233560000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc2442d79e0, 0xc0460c31b8, 0x9, 0x9, 0xc22f1f51f8, 0x461d40, 0xc22f1f51a0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc2442d79e0, 0xc0460c31b8, 0x9, 0x9, 0x9, 0xc0594e5d01, 0x10000000040cfa8, 0xc02d3b8e70)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c31b8, 0x9, 0x9, 0x2b41140, 0xc2442d79e0, 0x0, 0x0, 0x0, 0xc02d3b8e60)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3180, 0xc1a9c36770, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc52a933e00, 0xc22f1db620, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc52a933e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc52a933e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582542 [IO wait]:
internal/poll.runtime_pollWait(0x7f5887154450, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc23b12a018, 0x72, 0xe500, 0xe506, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc23b12a000, 0xc25b8b4000, 0xe506, 0xe506, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc23b12a000, 0xc25b8b4000, 0xe506, 0xe506, 0x203092, 0x420bb5, 0xc2494fa9a0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150304b90, 0xc25b8b4000, 0xe506, 0xe506, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc2494fa9a0, 0xc25b8b4000, 0xe506, 0xe506, 0x1f38, 0xe4f9, 0xc1afb0ee90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07ab76cd8, 0x2b41380, 0xc2494fa9a0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07ab76a80, 0x2b48520, 0xc150304b90, 0x5, 0xc150304b90, 0x1f28)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07ab76a80, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07ab76a80, 0xc2110fdab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0f78e2240, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc135863e60, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc23b13d720, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc00702a700, 0xc2110fdab2, 0x254e, 0x254e, 0x1f28, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc00702a700, 0xc2110fdab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc00702a780, 0xc2110fdab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc00702a780, 0xc2110f8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc052bfca20, 0xc052bfca80, 0xc052bfcae0, 0xc00d5805c0, 0xc20cd8d7d0, 0xc143aa7810, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4582837 [select]:
net/http.(*persistConn).writeLoop(0xc0fb8f86c0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4084802 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc06549cd80, 0xc223fa8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc147ddba40, 0xc223fa8000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc2079cb800, 0xc223fa8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc112f10050, 0x7f59e0599060, 0xc2079cb800, 0xc223fa8000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc112f10050, 0x7f59e0599060, 0xc2079cb800, 0xc223fa8000, 0x8000, 0x8000, 0x2b6fc80, 0xc2079cb800, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0052ce5c0, 0x2b44ea0, 0xc112f10050, 0xc223fa8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xafc070deee727601, 0x602716a9a5bef5f3, 0xc1bdd69600, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xafc070deee727601, 0x602716a9a5bef5f3, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xafc070deee727601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc001ae1e00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4501842 [semacquire]:
sync.runtime_Semacquire(0xc036d84fd0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc036d84fc8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc036d84fc0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1e5f75050)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc036d84f30, 0x255e040, 0x18, 0xc130c313f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc088e9d200, 0x2bbccc0, 0xc03785a900, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc088e9d200, 0x7f63bf19f960, 0xc0026ce730, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc088e9d200, 0x2bbbd60, 0xc0026ce730, 0xc000863100, 0xc240db7688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc03785a860, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc03785a840, 0xc03785a6e0, 0x26b3908, 0xc03785a840, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc03785a840, 0x2b99d40, 0xc036d84300)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc03785a820, 0xc03785a6e0, 0xc03785a700, 0x7f63e742f5d8, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc03785a820, 0xc03785a6e0, 0xc03785a700, 0x2b99d01, 0xc03785a820)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc03785a820, 0x2b99d40, 0xc036d842d0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0xc03785a720, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0x1, 0xc03785a6a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0xc03785a740, 0xdf79c1, 0xb5b1fc769c44)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0x20300d, 0x0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x7f59d9bbc400, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0xc03785a760, 0xc03785a6e0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc02ad75c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1d7dcd380, 0xc03785a6e0, 0x26b3908, 0x2b99d40, 0xc036d84270)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1664d0200, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1664d0200, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0b41fb330, 0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1664d0200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4555621 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc081fa6900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582080 [select]:
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc01ade3620, 0xc01ade3680, 0xc01ade36e0, 0xc009911e70, 0xc1e8866cc0, 0xc1068bf970, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4084487 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc2cfa40900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0xc1273fb5c0, 0x40cfa8, 0x60, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77
github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9
github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x0, 0x2000107, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b
github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc2570e5170, 0x25, 0x2886d48d, 0x26f0620, 0x0, 0x7f59e0599060, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a
github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc1f6a00220, 0x20, 0x2886d48d, 0x26f0620, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc0052ce1c0, 0x2b44ea0, 0xc107d13090, 0xc21fca6000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xfae3d9fe9f627601, 0xb0169f34b529a016, 0xc115d56100, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfae3d9fe9f627601, 0xb0169f34b529a016, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfae3d9fe9f627601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc002599200, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4514702 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582744 [runnable]:
internal/poll.runtime_pollWait(0x7f59c9c50468, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc20ac7f318, 0x72, 0xd400, 0xd401, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc20ac7f300, 0xc2492e0000, 0xd401, 0xd401, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc20ac7f300, 0xc2492e0000, 0xd401, 0xd401, 0x203094, 0x420bb5, 0xc25387a2a0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc007b7ef10, 0xc2492e0000, 0xd401, 0xd401, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc25387a2a0, 0xc2492e0000, 0xd401, 0xd401, 0x0, 0x41310b, 0xc02bbd9900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc0731953d8, 0x2b41380, 0xc25387a2a0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc073195180, 0x2b48520, 0xc007b7ef10, 0x5, 0xc007b7ef10, 0x40c924)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc073195180, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc073195180, 0xc23dd2f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0f84707e0, 0xc23dd2f000, 0x1000, 0x1000, 0xc072dc6060, 0xc02bbd9c58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc23e84e000)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc23e84e000, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0ca347440)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0f84707e0)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4517378 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1038bcf00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582303 [select]:
net/http.(*persistConn).writeLoop(0xc0fa6fe480)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4084805 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1f0dbad80, 0xc227da6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc11af77020, 0xc227da6000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1ea0b58f0, 0xc227da6000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc0fa984280, 0x7f59e0599060, 0xc1ea0b58f0, 0xc227da6000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc0fa984280, 0x7f59e0599060, 0xc1ea0b58f0, 0xc227da6000, 0x8000, 0x8000, 0x2b6fc80, 0xc1ea0b58f0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc14ee7f5c0, 0x2b44ea0, 0xc0fa984280, 0xc227da6000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xff02bb445c447701, 0x1e671c2618fcb70b, 0xc144281300, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xff02bb445c447701, 0x1e671c2618fcb70b, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xff02bb445c447701, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc007f18000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4581515 [select]:
net/http.(*persistConn).roundTrip(0xc0f84707e0, 0xc004d64340, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2559 +0x763
net/http.(*Transport).roundTrip(0xc000ce5540, 0xc12e007600, 0xc1f071a6c0, 0x40f967, 0xc0000874c8)
/usr/local/go/src/net/http/transport.go:582 +0xa88
net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc12e007600, 0xc000ce5540, 0x0, 0x0)
/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc12e007600, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc1c746b458, 0x2537f0f, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:252 +0x43e
net/http.(*Client).send(0xc000bfaba0, 0xc12e007600, 0x0, 0x0, 0x0, 0xc1c746b458, 0x0, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).do(0xc000bfaba0, 0xc12e007600, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:699 +0x44a
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:567
github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50
github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409
github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1863eaf60, 0xc1863eafc0, 0xc1863eb020, 0xc445ab91a0, 0xc192611710, 0xc109318e70, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4510179 [IO wait]:
internal/poll.runtime_pollWait(0x7f59dbd66458, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1ebaee118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1ebaee100, 0xc0c61c4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1ebaee100, 0xc0c61c4000, 0x8000, 0x8000, 0x203087, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004ffa028, 0xc0c61c4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc22a2638c0, 0xc12d6663b8, 0x9, 0x9, 0xc001c91500, 0x1007f63e7434368, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc22a2638c0, 0xc12d6663b8, 0x9, 0x9, 0x9, 0x601742e3, 0xc22c8bedc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc12d6663b8, 0x9, 0x9, 0x2b41140, 0xc22a2638c0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc12d666380, 0xc252fc4f20, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc19966b800, 0xc23e2b1a40, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc19966b800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc19966b800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582749 [select]:
net/http.(*persistConn).writeLoop(0xc0d0a00d80)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4514698 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8690, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d64e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4513294 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d9b65f18, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e400, 0xc14875c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e400, 0xc14875c000, 0x8000, 0x8000, 0x800010601, 0x0, 0x106010000008f)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d460b0, 0xc14875c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0ff161e60, 0xc00507c038, 0x9, 0x9, 0xc038e1ad40, 0x457243, 0xb5b93c2a95d0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0ff161e60, 0xc00507c038, 0x9, 0x9, 0x9, 0xbffdee99c8cb24a2, 0x15b79bf0db2, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00507c038, 0x9, 0x9, 0x2b41140, 0xc0ff161e60, 0x0, 0x0, 0xc25e9bd1e0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00507c000, 0xc25e9bd1e0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc065466f00, 0xc24859a210, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc065466f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc065466f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4515933 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc07661ae60, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19149f980, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00bc80f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582131 [semacquire]:
sync.runtime_SemacquireMutex(0xc00ddea134, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00ddea0b0, 0xc1580704d0, 0x8, 0xc25a00bc50, 0x21, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc197c69480, 0xc1ea9307a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc197c69480, 0xc00357c4c8, 0x1, 0x1, 0xc0d39ff800, 0xd, 0x1, 0xc001c16380, 0x7f63e741a848)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00618cb10, 0xc197c69480, 0xc13edbc7e0, 0xc00357c4c8, 0x1, 0x1, 0xc13edbc1e0, 0x2b44f80, 0xc1ea93a480, 0xc00017c0b8, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1ea93a450, 0xc1e8aaf570)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4464265 [semacquire, 3 minutes]:
sync.runtime_Semacquire(0xc238bf2be0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc238bf2bd8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc238bf2bd0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc239202960)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc238bf2b10, 0x255e040, 0x18, 0xc1ea4f13f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc238bf4600, 0x2bbccc0, 0xc067f04580, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc238bf4600, 0x7f63bf19f960, 0xc238bfc030, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc238bf4600, 0x2bbbd60, 0xc238bfc030, 0xc000863100, 0xc238e17688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc067f044c0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc067f04480, 0xc067f042a0, 0x26b3908, 0xc067f04480, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc067f04480, 0x2b99d40, 0xc238bf29c0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc067f04460, 0xc067f042a0, 0xc067f042e0, 0x7f63e743a180, 0xc0e13cfe00)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc067f04460, 0xc067f042a0, 0xc067f042e0, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc067f04460, 0x2b99d40, 0xc238bf2990)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0xc067f04300, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0x1, 0xc067f04260)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0xc067f04320, 0x293891ad, 0xb57b321d7c2c)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0x203019, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0xc067f04360, 0xc067f042a0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc0f0ed7c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc238bba0c0, 0xc067f042a0, 0x26b3908, 0x2b99d40, 0xc238bf2930)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0073fe780, 0xc238bbeb00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0073fe780, 0xc238bbeb00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc02c9b5d20, 0xc0002c3380, 0x2bc6d40, 0xc0073fe780, 0xc238bbeb00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4582327 [runnable]:
internal/poll.runtime_pollWait(0x7f59deb9d640, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1fffd3398, 0x72, 0xdd00, 0xddd6, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1fffd3380, 0xc249d50000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1fffd3380, 0xc249d50000, 0xddd6, 0xddd6, 0x203094, 0x420bb5, 0xc25273e740)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150ab4cf8, 0xc249d50000, 0xddd6, 0xddd6, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc25273e740, 0xc249d50000, 0xddd6, 0xddd6, 0x3668, 0x5555, 0xc0fffcae90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07a507758, 0x2b41380, 0xc25273e740, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07a507500, 0x2b48520, 0xc150ab4cf8, 0x5, 0xc150ab4cf8, 0x3658)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07a507500, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07a507500, 0xc223faebfa, 0x1406, 0x1406, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0d0a00d80, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc23e84e300, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc22509f560, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc1a42198c0, 0xc223faebfa, 0x1406, 0x1406, 0x3658, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc1a42198c0, 0xc223faebfa, 0x1406, 0x1406, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc1a4219900, 0xc223faebfa, 0x1406, 0x1406, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc1a4219900, 0xc223fa8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc16aa90ae0, 0xc16aa90b40, 0xc16aa90ba0, 0xc07df935c0, 0xc1ea0b5590, 0xc0a7228bb0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4581869 [semacquire]:
sync.runtime_Semacquire(0xc114311c90)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc114311c88)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc114311c80, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1b272fc80)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc114311b60, 0x255e040, 0x18, 0xc0f55573f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc121ce5920, 0x2bbccc0, 0xc1ea0a9d80, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc121ce5920, 0x7f63bf19f960, 0xc07df92710, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc121ce5920, 0x2bbbd60, 0xc07df92710, 0xc000863100, 0xc23f485688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1ea0a9ce0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea0a9cc0, 0xc1ea0a9b60, 0x26b3908, 0xc1ea0a9cc0, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1ea0a9cc0, 0x2b99d40, 0xc1143119b0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea0a9ca0, 0xc1ea0a9b60, 0xc1ea0a9b80, 0x7f63e742d0f8, 0xc00e0ad680)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea0a9ca0, 0xc1ea0a9b60, 0xc1ea0a9b80, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1ea0a9ca0, 0x2b99d40, 0xc114311980)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0xc1ea0a9ba0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0x1, 0xc1ea0a9b20)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0xc1ea0a9bc0, 0x2df8cb00, 0xb5b9d883f77b)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0x20307a, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0xc1ea0a9be0, 0xc1ea0a9b60, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc00211ac68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc180a25740, 0xc1ea0a9b60, 0x26b3908, 0x2b99d40, 0xc114311920)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc21d477500, 0xc1297e9800, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc21d477500, 0xc1297e9800, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc06d5bfa20, 0xc0002c3380, 0x2bc6d40, 0xc21d477500, 0xc1297e9800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4582173 [runnable]:
internal/poll.runtime_pollWait(0x7f5886ec6b98, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc19e32ff18, 0x72, 0xe300, 0xe36e, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc19e32ff00, 0xc25b8d0000, 0xe36e, 0xe36e, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc19e32ff00, 0xc25b8d0000, 0xe36e, 0xe36e, 0x203095, 0x420bb5, 0xc25668db40)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xca27c505c8, 0xc25b8d0000, 0xe36e, 0xe36e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc25668db40, 0xc25b8d0000, 0xe36e, 0xe36e, 0x2480, 0x5aed, 0xc228ecee90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc073b24958, 0x2b41380, 0xc25668db40, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc073b24700, 0x2b48520, 0xca27c505c8, 0x5, 0xca27c505c8, 0x2470)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc073b24700, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc073b24700, 0xc225e6bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc24422a900, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc14a10bf80, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc23c15d520, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc006618ec0, 0xc225e6bab2, 0x254e, 0x254e, 0x2470, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc006618ec0, 0xc225e6bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc006618f40, 0xc225e6bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc006618f40, 0xc225e66000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0636dfb60, 0xc0636dfc20, 0xc0636dfc80, 0xc2206abc80, 0xc1e9d93260, 0xc12a8244d0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4084803 [runnable]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc198ff8a80, 0xc22c7b0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc196b166c0, 0xc22c7b0000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc20f761ce0, 0xc22c7b0000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc0fda1fa40, 0x7f59e0599060, 0xc20f761ce0, 0xc22c7b0000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc0fda1fa40, 0x7f59e0599060, 0xc20f761ce0, 0xc22c7b0000, 0x8000, 0x8000, 0x2b6fc80, 0xc20f761ce0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1e07de8c0, 0x2b44ea0, 0xc0fda1fa40, 0xc22c7b0000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x56594bfb07bb7601, 0x17e3075b349f05d6, 0xc115bbe880, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x56594bfb07bb7601, 0x17e3075b349f05d6, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x56594bfb07bb7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc004abf800, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4522980 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc4f1083800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4084489 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1b69ccc00, 0xc21a218000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc1eddcf800, 0xc21a218000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc20deb1a70, 0xc21a218000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc102819310, 0x7f59e0599060, 0xc20deb1a70, 0xc21a218000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc102819310, 0x7f59e0599060, 0xc20deb1a70, 0xc21a218000, 0x8000, 0x8000, 0x2b6fc80, 0xc20deb1a70, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc006e3ab40, 0x2b44ea0, 0xc102819310, 0xc21a218000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xdd2142fb2cbd7601, 0x6f99466627d38173, 0xc1bdd68ec0, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdd2142fb2cbd7601, 0x6f99466627d38173, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdd2142fb2cbd7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0054ca000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4084488 [runnable]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1aa0cfb00, 0xc224632000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e49ae40, 0xc224632000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1a1b7dcb0, 0xc224632000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc112ec6b40, 0x7f59e0599060, 0xc1a1b7dcb0, 0xc224632000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc112ec6b40, 0x7f59e0599060, 0xc1a1b7dcb0, 0xc224632000, 0x8000, 0x8000, 0x2b6fc80, 0xc1a1b7dcb0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1e07de680, 0x2b44ea0, 0xc112ec6b40, 0xc224632000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x29e8bbfb721c7701, 0x15a357751cb03b9f, 0xc113afc380, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x29e8bbfb721c7701, 0x15a357751cb03b9f, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x29e8bbfb721c7701, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc004358900, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4582302 [select]:
net/http.(*persistConn).readLoop(0xc0fa6fe480)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4582838 [select]:
net/http.(*persistConn).readLoop(0xc0f7ded7a0)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4566155 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0ab132a80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4515974 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc027f04cd0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d9dfc4e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0de578000)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4561274 [select]:
net/http.(*persistConn).writeLoop(0xc27abf85a0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4570183 [semacquire]:
sync.runtime_SemacquireMutex(0xc00d68dc04, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00d68db80, 0xc0bad590b4, 0xb, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc23f3997d0, 0xc0e77e0ab0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc003b64ec0, 0xc0546f37c0, 0x3, 0x3, 0xc0b597b320, 0xc0b597b380, 0xc0b597b3e0, 0xc1379a6060, 0xc0f061a8a0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0077aba70, 0xc003b64ec0, 0xc149288900, 0xc0546f37c0, 0x3, 0x3, 0xc065e39800, 0x2b44f80, 0xc0e77e0c00, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0e77e0bd0, 0xc000233260)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4511299 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc167bf0900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582711 [select]:
net/http.(*persistConn).writeLoop(0xc0f41e47e0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4582834 [select]:
net/http.(*persistConn).readLoop(0xc10297e000)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4581516 [select]:
github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc1863eb080, 0xa, 0x2b99c80, 0xc004d641c0, 0xc1926117a0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c
created by github.com/minio/minio-go/v7.Client.newRetryTimer
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163
goroutine 4582216 [select]:
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1da35ade0, 0xc1da35ae40, 0xc1da35aea0, 0xc009b1d580, 0xc1a1f8def0, 0xc0df870840, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4582301 [select]:
net/http.(*persistConn).writeLoop(0xc9aa80f680)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4515567 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1437f97c0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1dd46eba0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc19caf6f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4577211 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15680f900, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc246348cc0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc52a6fd200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4514790 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8e60, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0910480, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc52a933e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4515930 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9f5ba40, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc166879098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc166879080, 0xc2473da000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc166879080, 0xc2473da000, 0x8000, 0x8000, 0x400000801, 0x1, 0x8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1505a40e8, 0xc2473da000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc19149ea20, 0xc1df964498, 0x9, 0x9, 0xc2473da034, 0x4, 0x18)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc19149ea20, 0xc1df964498, 0x9, 0x9, 0x9, 0xc000892dc8, 0x40cfa8, 0x20)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1df964498, 0x9, 0x9, 0x2b41140, 0xc19149ea20, 0x0, 0xc200000000, 0x2124f40, 0xc123f98470)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1df964460, 0xc123f98460, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00bc80d80, 0xc23ec67680, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc00bc80d80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc00bc80d80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4491184 [semacquire]:
sync.runtime_Semacquire(0xc245857870)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc245857868)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc245857860, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc23e33dbf0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc147c57c20, 0x255e040, 0x18, 0xc2460b93f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc182805b00, 0x2bbccc0, 0xc18585a000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc182805b00, 0x7f63bf19f960, 0xc14837f080, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc182805b00, 0x2bbbd60, 0xc14837f080, 0xc000863100, 0xc239f45688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc185855f60, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc185855f40, 0xc185855de0, 0x26b3908, 0xc185855f40, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc185855f40, 0x2b99d40, 0xc147c57ad0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc185855f20, 0xc185855de0, 0xc185855e00, 0x7f63e7430a30, 0xc0231bed80)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc185855f20, 0xc185855de0, 0xc185855e00, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc185855f20, 0x2b99d40, 0xc147c57aa0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0xc185855e20, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0x1, 0xc185855da0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0xc185855e40, 0x19049ca, 0xb59d44573248)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0x203061, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0xc185855e60, 0xc185855de0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc0c1dc4c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc006ffe180, 0xc185855de0, 0x26b3908, 0x2b99d40, 0xc147c57a40)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876200, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876200, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0182a84c0, 0xc0002c3380, 0x2bc6d40, 0xc028cbc480, 0xc1e6876200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4084484 [syscall]:
syscall.Syscall(0x4a, 0x249, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5
syscall.Fsync(0x249, 0x1, 0x14)
/usr/local/go/src/syscall/zsyscall_linux_amd64.go:413 +0x40
internal/poll.(*FD).Fsync(0xc13fee3620, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_fsync_posix.go:17 +0x92
os.(*File).Sync(0xc151b91520, 0x0, 0x0)
/usr/local/go/src/os/file_posix.go:113 +0x4c
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xc0ef16aae2b87601, 0x2b82198c9bb43095, 0xc1338effc0, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:116 +0x421
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xc0ef16aae2b87601, 0x2b82198c9bb43095, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xc0ef16aae2b87601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0036a2b00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4474340 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d10efa90, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d6757320, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1918b0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4084490 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1e93b1e00, 0xc2110f8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc147ddb920, 0xc2110f8000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc2079cb5f0, 0xc2110f8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc0b3b094a0, 0x7f59e0599060, 0xc2079cb5f0, 0xc2110f8000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc0b3b094a0, 0x7f59e0599060, 0xc2079cb5f0, 0xc2110f8000, 0x8000, 0x8000, 0x2b6fc80, 0xc2079cb5f0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc004d14600, 0x2b44ea0, 0xc0b3b094a0, 0xc2110f8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xaca6514cb73a7601, 0x1d35228a8b6b854, 0xc113353580, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xaca6514cb73a7601, 0x1d35228a8b6b854, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xaca6514cb73a7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0010c8b00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4578952 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc079945680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4519239 [semacquire]:
sync.runtime_Semacquire(0xc287ebdd50)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc287ebdd48)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc287ebdd40, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc28ab641b0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc131f367e0, 0x255e040, 0x18, 0xc2401353f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1d70ea660, 0x2bbccc0, 0xc0e07f5240, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1d70ea660, 0x7f63bf19f960, 0xc24318a320, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1d70ea660, 0x2bbbd60, 0xc24318a320, 0xc000863100, 0xc240135688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0e07f51a0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0e07f5180, 0xc0e07f5020, 0x26b3908, 0xc0e07f5180, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0e07f5180, 0x2b99d40, 0xc131f36690)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0e07f5160, 0xc0e07f5020, 0xc0e07f5040, 0x7f63e74235d8, 0xc0bc7c8300)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0e07f5160, 0xc0e07f5020, 0xc0e07f5040, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0e07f5160, 0x2b99d40, 0xc131f36660)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0xc0e07f5060, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0x1, 0xc0e07f4fe0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0xc0e07f5080, 0x25b4f230, 0xb5b5daf8b4c0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0x203038, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0xc0e07f50a0, 0xc0e07f5020, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0fb153c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc130274f00, 0xc0e07f5020, 0x26b3908, 0x2b99d40, 0xc131f36600)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc106a9d600, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc106a9d600, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc126847560, 0xc0002c3380, 0x2bc6d40, 0xc028eec180, 0xc106a9d600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4474342 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a90be5a48, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc239f57b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc239f57b00, 0xc1a0272000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc239f57b00, 0xc1a0272000, 0x8000, 0x8000, 0x4647d0, 0xc166433c80, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc151594118, 0xc1a0272000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d6757260, 0xc1febc10d8, 0x9, 0x9, 0xc0d3534e98, 0x461d40, 0xc0d3534e40)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d6757260, 0xc1febc10d8, 0x9, 0x9, 0x9, 0xc0f4bc9d01, 0x10000000040cfa8, 0xc0d10efaa0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1febc10d8, 0x9, 0x9, 0x2b41140, 0xc1d6757260, 0x0, 0x0, 0x0, 0xc0d10efa90)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1febc10a0, 0xc25c2e6310, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1918b0600, 0xc1d7dc8390, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1918b0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1918b0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4581666 [running]:
goroutine running on other thread; stack unavailable
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4513289 [IO wait]:
internal/poll.runtime_pollWait(0x7f5886ec7298, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1074ca118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1074ca100, 0xc0e956c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1074ca100, 0xc0e956c000, 0x8000, 0x8000, 0x20309d, 0x420bb5, 0xc2747746e0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000d54030, 0xc0e956c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc19679f200, 0xc0f0814ab8, 0x9, 0x9, 0xc005ac0000, 0x1007f63e74357c0, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc19679f200, 0xc0f0814ab8, 0x9, 0x9, 0x9, 0x601742dd, 0xc14a0bedc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0f0814ab8, 0x9, 0x9, 0x2b41140, 0xc19679f200, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f0814a80, 0xc076bf4760, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc02f0d4600, 0xc00c2cf0e0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc02f0d4600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc02f0d4600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4515944 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d62e1590, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d8e1e2a0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc028eec180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4388231 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc0f5e21e70)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0f5e21e68)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc0f5e21e60, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0ff4119b0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0f5e21da0, 0x255e040, 0x18, 0xc0f1f2f3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc0f5e42780, 0x2bbccc0, 0xc068e64a60, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc0f5e42780, 0x7f63bf19f960, 0xc0f5e6c200, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc0f5e42780, 0x2bbbd60, 0xc0f5e6c200, 0xc000863100, 0xc0d956b688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc068e649c0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc068e649a0, 0xc068e64840, 0x26b3908, 0xc068e649a0, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc068e649a0, 0x2b99d40, 0xc0f5e21c50)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc068e64980, 0xc068e64840, 0xc068e64860, 0x7f63e7422f10, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc068e64980, 0xc068e64840, 0xc068e64860, 0x2b99d01, 0xc068e64980)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc068e64980, 0x2b99d40, 0xc0f5e21c20)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0xc068e64880, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0x1, 0xc068e64800)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0xc068e648a0, 0x1d76e66a, 0xb579c0bb14e5)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0x20301a, 0x8701ba)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0xc068e648c0, 0xc068e64840, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc12b027c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc04d9e75c0, 0xc068e64840, 0x26b3908, 0x2b99d40, 0xc0f5e21bc0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0158e2480, 0xc0f5e0bc00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0158e2480, 0xc0f5e0bc00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0b0cfd630, 0xc0002c3380, 0x2bc6d40, 0xc0158e2480, 0xc0f5e0bc00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4582840 [select]:
net/http.(*persistConn).readLoop(0xc24422a7e0)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4516668 [IO wait]:
internal/poll.runtime_pollWait(0x7f63bf05ed88, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0d4a11b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d4a11b00, 0xc249970000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0d4a11b00, 0xc249970000, 0x8000, 0x8000, 0x20301d, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0b7aa0640, 0xc249970000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc09f936960, 0xc12d667ed8, 0x9, 0x9, 0xc009779880, 0x1007f63e743bca0, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc09f936960, 0xc12d667ed8, 0x9, 0x9, 0x9, 0x601742e3, 0xc079ecbdc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc12d667ed8, 0x9, 0x9, 0x2b41140, 0xc09f936960, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc12d667ea0, 0xc0db948080, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc2fd38d200, 0xc24994faa0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc2fd38d200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc2fd38d200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4566156 [IO wait]:
internal/poll.runtime_pollWait(0x7f588a55e480, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0b4977a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0b4977a00, 0xc2462c0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0b4977a00, 0xc2462c0000, 0x8000, 0x8000, 0x400000801, 0xc000000001, 0x8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00df12120, 0xc2462c0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1acbcfce0, 0xc11853d8b8, 0x9, 0x9, 0xc1ec071740, 0xc0a9813df7, 0xc0a9813d80)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1acbcfce0, 0xc11853d8b8, 0x9, 0x9, 0x9, 0x406594, 0xc083742480, 0xc0a9813df7)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc11853d8b8, 0x9, 0x9, 0x2b41140, 0xc1acbcfce0, 0x0, 0xc000000000, 0x2124f40, 0xc0f44c47b8)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc11853d880, 0xc0f44c4800, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0ab132a80, 0xc1db6c9ad0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0ab132a80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0ab132a80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4389190 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc075ccc5f0, 0x8, 0xc081dd5ad0, 0x28, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc197c68440, 0xc09d4d34a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc197c68440, 0xc4168e2048, 0x1, 0x1, 0xc0b58b21e0, 0xd, 0x1, 0xc009779880, 0x7f63e743b5d8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc197c68440, 0xc100490720, 0xc4168e2048, 0x1, 0x1, 0xc1f17c4000, 0x2b44f80, 0xc1f17b26c0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc0284e8da0, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1f17b2690, 0xc0005abab0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4514798 [IO wait]:
internal/poll.runtime_pollWait(0x7f5881ab49f0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281c18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281c00, 0xc234290000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281c00, 0xc234290000, 0x8000, 0x8000, 0x4647d0, 0xc00c4a7380, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa460, 0xc234290000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0c0910a80, 0xc0460c3378, 0x9, 0x9, 0xc22f1f54f8, 0x461d40, 0xc22f1f54a0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0c0910a80, 0xc0460c3378, 0x9, 0x9, 0x9, 0xc0594e7d01, 0x10000000040cfa8, 0xc02d3b92d0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c3378, 0x9, 0x9, 0x2b41140, 0xc0c0910a80, 0x0, 0x0, 0x0, 0xc02d3b92c0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3340, 0xc114c7b480, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430780, 0xc22f1dba40, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4508535 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c032f5c0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc23a318098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc23a318080, 0xc2210c9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc23a318080, 0xc2210c9000, 0x1000, 0x1000, 0x4378cc, 0xc0fe5b6b58, 0x4647d0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc9f9b34098, 0xc2210c9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
net/http.(*persistConn).Read(0xc0deef0a20, 0xc2210c9000, 0x1000, 0x1000, 0xc093906780, 0xc0fe5b6c58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc1da7932c0)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc1da7932c0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc19ef33b60)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0deef0a20)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4582525 [select]:
github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0556e3b60, 0xa, 0x2b99c80, 0xc00702a880, 0xc2079cb920)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c
created by github.com/minio/minio-go/v7.Client.newRetryTimer
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163
goroutine 4565088 [IO wait]:
internal/poll.runtime_pollWait(0x7f588d6e5898, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00d100a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00d100a00, 0xc2910f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00d100a00, 0xc2910f4000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000071)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc9f9b34388, 0xc2910f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc11a5767e0, 0xc072428818, 0x9, 0x9, 0xc154428d40, 0x457243, 0xb5b93bba3c7f)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc11a5767e0, 0xc072428818, 0x9, 0x9, 0x9, 0xbffdee99c860543a, 0x15b79543d4d, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc072428818, 0x9, 0x9, 0x2b41140, 0xc11a5767e0, 0x0, 0x0, 0xc1fd836e20, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0724287e0, 0xc1fd836e20, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1ecc2c480, 0xc2910d0540, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1ecc2c480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1ecc2c480)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4511256 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1245db680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582644 [runnable]:
internal/poll.runtime_pollWait(0x7f5887ee0150, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1ea0b9498, 0x72, 0xe100, 0xe1c1, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1ea0b9480, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1ea0b9480, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x203092, 0x420bb5, 0xc2495d8de0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150ab4d00, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc2495d8de0, 0xc25b83c000, 0xe1c1, 0xe1c1, 0x3360, 0x5940, 0xc1273f6e90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07f222258, 0x2b41380, 0xc2495d8de0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07f222000, 0x2b48520, 0xc150ab4d00, 0x5, 0xc150ab4d00, 0x3350)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07f222000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07f222000, 0xc1a1b6b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc9aa80f680, 0xc1a1b6b000, 0x1000, 0x1000, 0x3350, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc23e49a180, 0xc20e237fa2, 0x5e, 0x5e, 0x3350, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.(*LimitedReader).Read(0xc23b0f8240, 0xc20e237fa2, 0x5e, 0x5e, 0x3350, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc00130c780, 0xc20e237fa2, 0x5e, 0x5e, 0x3350, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc00130c780, 0xc20e237fa2, 0x5e, 0x5e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc00130c7c0, 0xc20e237fa2, 0x5e, 0x5e, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc00130c7c0, 0xc20e230000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc2654fa6c0, 0xc2654fa720, 0xc2654fa780, 0xc1ca6c62c0, 0xc20db2a9f0, 0xc085ceb1e0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4578951 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0fe80c000, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19958b0e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc079945680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4578949 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1e65f3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4515568 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc19caf6f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4559555 [semacquire]:
sync.runtime_SemacquireMutex(0xc00d68dc04, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00d68db80, 0xc1f2d70c84, 0xb, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc1f01ed7d0, 0xc287ebdc20, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc005a83900, 0xc28b153b00, 0x3, 0x3, 0xc1335a85f0, 0xd, 0x1, 0xc222d2dc00, 0x7f63e74390f8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0077aba70, 0xc005a83900, 0xc1358474a0, 0xc28b153b00, 0x3, 0x3, 0xc1d70ea660, 0x2b44f80, 0xc287ebdd70, 0x8c, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc2837a3880, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc287ebdd40, 0xc14ebd24d0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4084806 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc21558cc00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0xc1e8a935c0, 0x40cfa8, 0x60, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77
github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9
github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x0, 0x4000105, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b
github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc257ace030, 0x25, 0x19cf51f4, 0x1c5f43a, 0x0, 0x7f59e0599060, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a
github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc254da6d60, 0x20, 0x19cf51f4, 0x1c5f43a, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc00266bf40, 0x2b44ea0, 0xc114f2ae60, 0xc210bdc000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x425ec0b5923b7601, 0x158267717d66e356, 0xc12deb8f00, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x425ec0b5923b7601, 0x158267717d66e356, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x425ec0b5923b7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc002343700, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4561862 [IO wait]:
internal/poll.runtime_pollWait(0x7f59a29f3388, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc28cf02e98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc28cf02e80, 0xc13f998000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc28cf02e80, 0xc13f998000, 0x8000, 0x8000, 0x20307a, 0x420bb5, 0xc1ea4b6a00)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc46e0a2028, 0xc13f998000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1908b5aa0, 0xc112bf4038, 0x9, 0x9, 0xc001e05180, 0x1007f63e74310f8, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1908b5aa0, 0xc112bf4038, 0x9, 0x9, 0x9, 0x601742ea, 0xc160c70dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc112bf4038, 0x9, 0x9, 0x2b41140, 0xc1908b5aa0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc112bf4000, 0xc2561c5800, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0ab2bbe00, 0xc28f847470, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc0ab2bbe00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc0ab2bbe00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4510186 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc19966bb00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582808 [select]:
github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc092731ec0, 0xa, 0x2b99c80, 0xc004f95480, 0xc23b892f00)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c
created by github.com/minio/minio-go/v7.Client.newRetryTimer
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163
goroutine 4514789 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d957e260, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1c30b4298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1c30b4280, 0xc233548000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1c30b4280, 0xc233548000, 0x8000, 0x8000, 0x203048, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1f017c018, 0xc233548000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc2442d73e0, 0xc0460c30d8, 0x9, 0x9, 0xc1d4e89c00, 0x1007f63e741a848, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc2442d73e0, 0xc0460c30d8, 0x9, 0x9, 0x9, 0x601742e3, 0xc07159bdc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c30d8, 0x9, 0x9, 0x2b41140, 0xc2442d73e0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c30a0, 0xc260cdd5c0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc52a933c80, 0xc22f1db410, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc52a933c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc52a933c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4581580 [running]:
goroutine running on other thread; stack unavailable
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4517322 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9c987d8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1c30b4418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1c30b4400, 0xc202fbe000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1c30b4400, 0xc202fbe000, 0x8000, 0x8000, 0x800010601, 0x0, 0x10601000000f8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1f017c030, 0xc202fbe000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1df74d320, 0xc0ce7d9618, 0x9, 0x9, 0xc038e1dd40, 0x457243, 0xb5b93caf9027)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1df74d320, 0xc0ce7d9618, 0x9, 0x9, 0x9, 0xbffdee99c94efb94, 0x15b7a42e4dc, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0ce7d9618, 0x9, 0x9, 0x2b41140, 0xc1df74d320, 0x0, 0x0, 0xc0a0e95f00, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0ce7d95e0, 0xc0a0e95f00, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a2f8e780, 0xc23f952f90, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a2f8e780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4084486 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc19966a300, 0xc225e66000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e84e5a0, 0xc225e66000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1e8b47ad0, 0xc225e66000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc0b8635220, 0x7f59e0599060, 0xc1e8b47ad0, 0xc225e66000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc0b8635220, 0x7f59e0599060, 0xc1e8b47ad0, 0xc225e66000, 0x8000, 0x8000, 0x2b6fc80, 0xc1e8b47ad0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1e07de4c0, 0x2b44ea0, 0xc0b8635220, 0xc225e66000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x2e1ca4e423a27601, 0x194080b5c8660802, 0xc11632e400, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2e1ca4e423a27601, 0x194080b5c8660802, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x2e1ca4e423a27601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc002413f00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4582841 [select]:
net/http.(*persistConn).writeLoop(0xc24422a7e0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4513605 [IO wait]:
internal/poll.runtime_pollWait(0x7f58877a53c8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e518, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e500, 0xc103990000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e500, 0xc103990000, 0x8000, 0x8000, 0x4647d0, 0xc00109f080, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d460c0, 0xc103990000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc10cfe4f00, 0xc1b5e6c578, 0x9, 0x9, 0xc2469ad978, 0x461d40, 0xc2469ad920)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc10cfe4f00, 0xc1b5e6c578, 0x9, 0x9, 0x9, 0xc079eccd01, 0x10000000040cfa8, 0xc175158d80)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc1b5e6c578, 0x9, 0x9, 0x2b41140, 0xc10cfe4f00, 0x0, 0x0, 0x0, 0xc175158d70)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc1b5e6c540, 0xc201589ff0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc10a44af00, 0xc24500a5d0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc10a44af00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc10a44af00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582745 [select]:
net/http.(*persistConn).writeLoop(0xc0f84707e0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4517320 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0df651f90, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1cb011f80, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a2f8e780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4514915 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1503a0500, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc198be6c60, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4513288 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc02f0d4600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4514701 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b8870, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d6960, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477c80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4513287 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc157075040, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19679f2c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc02f0d4600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582635 [select]:
net/http.(*persistConn).readLoop(0xc2386e6a20)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4517361 [IO wait]:
internal/poll.runtime_pollWait(0x7f58927f8ee0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e480, 0xc147c9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e480, 0xc147c9a000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7bacee0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d460b8, 0xc147c9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc100420060, 0xc0c822c658, 0x9, 0x9, 0xc009778a80, 0x1007f63e7434368, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc100420060, 0xc0c822c658, 0x9, 0x9, 0x9, 0x601742ea, 0xc038e20dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0c822c658, 0x9, 0x9, 0x2b41140, 0xc100420060, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c822c620, 0xc1f810baa0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc19caf6f00, 0xc24726b0b0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc19caf6f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc19caf6f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4556670 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc28317d9f0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc191227380, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc183057e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582712 [runnable]:
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc06ed7ab40, 0xc06ed7ade0, 0xc06ed7ae40, 0xc00d5e55c0, 0xc23ad14450, 0xc0099ab4a0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4514690 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0bdda9c70, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc2442d6060, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21d477500)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4084496 [syscall]:
syscall.Syscall(0x4a, 0x1d8, 0x0, 0x0, 0x4edbd3, 0xc1d72be380, 0xc249195778)
/usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5
syscall.Fsync(0x1d8, 0x2ade801, 0x0)
/usr/local/go/src/syscall/zsyscall_linux_amd64.go:413 +0x40
internal/poll.(*FD).Fsync(0xc193e528a0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_fsync_posix.go:17 +0x92
os.(*File).Sync(0xc029070e30, 0x3b, 0x8000)
/usr/local/go/src/os/file_posix.go:113 +0x4c
github.com/thanos-io/thanos/pkg/block/indexheader.newBinaryWriter(0xc1d72be380, 0x3b, 0xc2557ba000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:269 +0x22b
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xb15fcd643c807601, 0x4bc47f928c0a90ae, 0xc175b71200, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:82 +0x18b
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb15fcd643c807601, 0x4bc47f928c0a90ae, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb15fcd643c807601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc003dbae00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4582515 [runnable]:
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0167b2e40, 0xc0167b2ea0, 0xc0167b2f00, 0xc0d318b390, 0xc2079ca990, 0xc133c34580, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4513532 [IO wait]:
internal/poll.runtime_pollWait(0x7f63bf30f008, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc182359898, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc182359880, 0xc17e9c2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc182359880, 0xc17e9c2000, 0x8000, 0x8000, 0x800010601, 0x7f5900000000, 0x106010000006e)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0054060b8, 0xc17e9c2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc18aab5740, 0xc0dceeddf8, 0x9, 0x9, 0xc13e423d40, 0x457243, 0xb5b9026248b4)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc18aab5740, 0xc0dceeddf8, 0x9, 0x9, 0x9, 0xbffdee9985965cd7, 0x15b3aef7be7, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0dceeddf8, 0x9, 0x9, 0x2b41140, 0xc18aab5740, 0x0, 0x0, 0xc2594b4260, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0dceeddc0, 0xc2594b4260, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc078bd3200, 0xc11b1a3740, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc078bd3200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc078bd3200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4521104 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc12e0d6300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4578957 [IO wait]:
internal/poll.runtime_pollWait(0x7f59d9b66378, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc167221a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc167221a80, 0xc1ad3da000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc167221a80, 0xc1ad3da000, 0x8000, 0x8000, 0x10401, 0x0, 0x1040102ade864)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc142924920, 0xc1ad3da000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1ea5010e0, 0xc06785a578, 0x9, 0x9, 0xc00596d180, 0x7f63e7434a30, 0xc04a5d1380)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1ea5010e0, 0xc06785a578, 0x9, 0x9, 0x9, 0xa6fc25, 0xc1136af5f0, 0xc1136a0004)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc06785a578, 0x9, 0x9, 0x2b41140, 0xc1ea5010e0, 0x0, 0x0, 0xc1136af5f0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc06785a540, 0xc1136af5f0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc079945980, 0xc19a278c30, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc079945980)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc079945980)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4516940 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc20dacfae0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1fd070060, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc003a9e900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4462380 [semacquire]:
sync.runtime_Semacquire(0xc1222108b0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1222108a8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1222108a0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1dc0d7b60)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc14bcc4870, 0x255e040, 0x18, 0xc2461153f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1549fe360, 0x2bbccc0, 0xc18983e260, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1549fe360, 0x7f63bf19f960, 0xc16eeb35d0, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1549fe360, 0x2bbbd60, 0xc16eeb35d0, 0xc000863100, 0xc238491688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc18b801fc0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18b801fa0, 0xc18b801e40, 0x26b3908, 0xc18b801fa0, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc18b801fa0, 0x2b99d40, 0xc13ffe2570)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18b801f80, 0xc18b801e40, 0xc18b801e60, 0x7f63e74250f8, 0xc054bff680)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18b801f80, 0xc18b801e40, 0xc18b801e60, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc18b801f80, 0x2b99d40, 0xc13ffe2540)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0xc18b801e80, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0x1, 0xc18b801e00)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0xc18b801ea0, 0x1bd9e773, 0xb59cabd071f1)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0x203062, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0xc18b801e00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0xc18b801ec0, 0x203062, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc027804c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0a17be0c0, 0xc18b801e40, 0x26b3908, 0x2b99d40, 0xc13ffe24e0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1defd8100, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1defd8100, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0b41fb330, 0xc0002c3380, 0x2bc6d40, 0xc0e542a480, 0xc1defd8100)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4562285 [semacquire]:
sync.runtime_SemacquireMutex(0xc007cd0344, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc007cd02c0, 0xc1580704d0, 0x8, 0xc25a00bc50, 0x21, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc197c692c0, 0xc2003ed1e0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc197c692c0, 0xc00357c4c8, 0x1, 0x1, 0x2d850e0a0cd5be5, 0x6b4423492606b1b3, 0xeb462cad18d3a9f, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0040c6990, 0xc197c692c0, 0xc13edbc3c0, 0xc00357c4c8, 0x1, 0x1, 0xc13edbc1e0, 0x2b44f80, 0xc1ea93a480, 0x60, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1ea93a450, 0xc1e8aaf2d0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582881 [select]:
github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc07f961ce0, 0xa, 0x2b99c80, 0xc1e07def00, 0xc225a73d10)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c
created by github.com/minio/minio-go/v7.Client.newRetryTimer
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163
goroutine 4522979 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc039d83ef0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1f9321380, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc4f1083800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582865 [select]:
net/http.(*persistConn).writeLoop(0xc22ca94a20)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4486231 [select, 3 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1c1c82180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582720 [select]:
net/http.(*persistConn).readLoop(0xc22ca94a20)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4582300 [select]:
net/http.(*persistConn).readLoop(0xc9aa80f680)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4582521 [select]:
net/http.(*persistConn).writeLoop(0xc0fd6c8000)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4514914 [IO wait]:
internal/poll.runtime_pollWait(0x7f63bf05e5a8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281c80, 0xc236b32000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281c80, 0xc236b32000, 0x8000, 0x8000, 0x4647d0, 0xc0b9286900, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa468, 0xc236b32000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0c09119e0, 0xc0460c3458, 0x9, 0x9, 0xc22f1f5738, 0x461d40, 0xc22f1f56e0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0c09119e0, 0xc0460c3458, 0x9, 0x9, 0x9, 0xc0594e8d01, 0x10000000040cfa8, 0xc1503a02e0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c3458, 0x9, 0x9, 0x2b41140, 0xc0c09119e0, 0x0, 0x0, 0x0, 0xc1503a02d0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3420, 0xc0f0f4bb70, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430c00, 0xc236b302a0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430c00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430c00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4514646 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4084493 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1c1797b00, 0xc224286000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc147ddb8c0, 0xc224286000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc2079cb4d0, 0xc224286000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc0bf1485a0, 0x7f59e0599060, 0xc2079cb4d0, 0xc224286000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc0bf1485a0, 0x7f59e0599060, 0xc2079cb4d0, 0xc224286000, 0x8000, 0x8000, 0x2b6fc80, 0xc2079cb4d0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc9f4139280, 0x2b44ea0, 0xc0bf1485a0, 0xc224286000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xd5aa35a62ec7601, 0x36cc35e8daaa0f55, 0xc115bc4240, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xd5aa35a62ec7601, 0x36cc35e8daaa0f55, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xd5aa35a62ec7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0073c2200, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4582668 [runnable]:
internal/poll.runtime_pollWait(0x7f59df27c0e8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1e9de4718, 0x72, 0xe000, 0xe0a5, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1e9de4700, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1e9de4700, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x203094, 0x420bb5, 0xc25273e980)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc142924a10, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc25273e980, 0xc25bbde000, 0xe0a5, 0xe0a5, 0x0, 0x41310b, 0xc145a15900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc066d7a258, 0x2b41380, 0xc25273e980, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc066d7a000, 0x2b48520, 0xc142924a10, 0x5, 0xc142924a10, 0x40c924)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc066d7a000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc066d7a000, 0xc23df34000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc163269d40, 0xc23df34000, 0x1000, 0x1000, 0xc0cc95a780, 0xc145a15c58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc0bad13e60)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0bad13e60, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc029bd1a40)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc163269d40)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4557186 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc11159f080)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4517270 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc20dacfbd0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1f71bac00, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc003a9ea80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4517321 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a2f8e780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4084485 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc01c240600, 0xc210d60000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e49b1a0, 0xc210d60000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1a1b7def0, 0xc210d60000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc11357b130, 0x7f59e0599060, 0xc1a1b7def0, 0xc210d60000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc11357b130, 0x7f59e0599060, 0xc1a1b7def0, 0xc210d60000, 0x8000, 0x8000, 0x2b6fc80, 0xc1a1b7def0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1a4218c40, 0x2b44ea0, 0xc11357b130, 0xc210d60000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xb3deb7395f797601, 0x6912f1f70bc8663, 0xc113352840, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb3deb7395f797601, 0x6912f1f70bc8663, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xb3deb7395f797601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0029ddd00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4514917 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9f7d630, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281d18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281d00, 0xc236b62000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281d00, 0xc236b62000, 0x8000, 0x8000, 0x800010601, 0x0, 0x106010000004c)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa470, 0xc236b62000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0c0911b00, 0xc0460c3618, 0x9, 0x9, 0xc079ecad40, 0x457243, 0xb5b93bf2a674)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0c0911b00, 0xc0460c3618, 0x9, 0x9, 0x9, 0xbffdee99c899aeb9, 0x15b798d97bc, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c3618, 0x9, 0x9, 0x2b41140, 0xc0c0911b00, 0x0, 0x0, 0xc0e6406980, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c35e0, 0xc0e6406980, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430f00, 0xc236b30570, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4508843 [IO wait]:
internal/poll.runtime_pollWait(0x7f5889dba5b8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e200, 0xc203e18000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e200, 0xc203e18000, 0x8000, 0x8000, 0x203034, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d46070, 0xc203e18000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d98a5c80, 0xc0f7ba02d8, 0x9, 0x9, 0xc22c4eca80, 0x1007f63e743f5d8, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d98a5c80, 0xc0f7ba02d8, 0x9, 0x9, 0x9, 0x601742e2, 0xc0594e4dc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0f7ba02d8, 0x9, 0x9, 0x2b41140, 0xc1d98a5c80, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f7ba02a0, 0xc0caf8a8e0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc4b81e0600, 0xc14938b920, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc4b81e0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc4b81e0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4084807 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1f0dbbe00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0xc23cc6f5c0, 0x40cfa8, 0x60, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77
github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9
github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x0, 0x1000100, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b
github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc257ab8750, 0x25, 0x15803a8f, 0x1d2ece6, 0x0, 0x7f59e0599060, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a
github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc253dc2f40, 0x20, 0x15803a8f, 0x1d2ece6, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc43967ff80, 0x2b44ea0, 0xc1271b6640, 0xc21b34c000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xdae8b344428f7601, 0x591094f781cff076, 0xc114726e40, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdae8b344428f7601, 0x591094f781cff076, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xdae8b344428f7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0036a8300, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4467634 [semacquire]:
sync.runtime_Semacquire(0xc1e00b9a20)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1e00b9a18)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1e00b9a10, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1edc66030)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc232a72f90, 0x255e040, 0x18, 0xc1e8c9b3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc2016bb560, 0x2bbccc0, 0xc1e66760e0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc2016bb560, 0x7f63bf19f960, 0xc2016b9500, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc2016bb560, 0x2bbbd60, 0xc2016b9500, 0xc000863100, 0xc169933688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1e6676040, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e6676000, 0xc1b5375ea0, 0x26b3908, 0xc1e6676000, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e6676000, 0x2b99d40, 0xc232a72e40)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375fe0, 0xc1b5375ea0, 0xc1b5375ec0, 0x7f63e741f5d8, 0xc04a91cc00)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375fe0, 0xc1b5375ea0, 0xc1b5375ec0, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375fe0, 0x2b99d40, 0xc232a72e10)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0xc1b5375ee0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0x1, 0xc1b5375e60)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0xc1b5375f00, 0x2a336da9, 0xb57d4b897222)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0x20306d, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0xc1b5375f20, 0xc1b5375ea0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc2271e5c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346af00, 0xc1b5375ea0, 0x26b3908, 0x2b99d40, 0xc232a72db0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469d00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469d00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0051527a0, 0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469d00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4514797 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430780)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4388342 [semacquire]:
sync.runtime_SemacquireMutex(0xc00410104c, 0x3f75800, 0x1)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*Mutex).lockSlow(0xc004101048)
/usr/local/go/src/sync/mutex.go:138 +0xfc
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0xc004101048)
/usr/local/go/src/sync/rwmutex.go:98 +0x97
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc004100fd0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).IndexVersion(0xc004100fd0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:144 +0xa0
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1f7072f00, 0xc9ea0960d8, 0x1, 0x1, 0xc026b230c0, 0xd, 0x1, 0xc000e7c700, 0x7f63e741c368)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1639 +0xfbf
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc002a7a1b0, 0xc1f7072f00, 0xc0ff2141e0, 0xc9ea0960d8, 0x1, 0x1, 0xc1c9270d80, 0x2b44f80, 0xc0ff208000, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0f4f5bfb0, 0xc0095a88c0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4514913 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1a3430c00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4511298 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc129f259a0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1d8e720c0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc167bf0900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4496797 [semacquire]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0xc23886c200, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00c51d1e0, 0xc18e916c60, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc0070d77d0, 0xc1e00b98f0, 0x66, 0x3f96db8, 0x203011)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc20a83d000, 0xc1ffe7c800, 0x2, 0x2, 0x203079, 0x417a5e, 0x7f59d8e90040, 0x203079, 0xbffdee95db925928)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc20a83d000, 0xc22ff38cc0, 0xc1ffe7c800, 0x2, 0x2, 0xc2016bb560, 0x2b44f80, 0xc1e00b9a40, 0xc00017c0b8, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e00b9a10, 0xc1307d1f80)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4514785 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4555622 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9d0ce48, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc125066318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc125066300, 0xc1deb1e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc125066300, 0xc1deb1e000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7b8be80)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc9f024c3f8, 0xc1deb1e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0056b9200, 0xc0f09056f8, 0x9, 0x9, 0xc1ed8d7880, 0x1007f63e7434368, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0056b9200, 0xc0f09056f8, 0x9, 0x9, 0x9, 0x601742e8, 0xc227d33dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0f09056f8, 0x9, 0x9, 0x2b41140, 0xc0056b9200, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f09056c0, 0xc1f810b740, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc081fa6900, 0xc23a5bb140, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc081fa6900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc081fa6900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4555620 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0d06739a0, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc093978de0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc081fa6900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582667 [select]:
net/http.(*persistConn).writeLoop(0xc0b6675440)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4521103 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0ea4f9f90, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ecefa900, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc12e0d6300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582524 [select]:
net/http.(*persistConn).roundTrip(0xc163269d40, 0xc00702a9c0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2559 +0x763
net/http.(*Transport).roundTrip(0xc000ce5540, 0xc249c4e900, 0xc249f206c0, 0x40f967, 0xc000140a08)
/usr/local/go/src/net/http/transport.go:582 +0xa88
net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc249c4e900, 0xc000ce5540, 0x0, 0x0)
/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc249c4e900, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc143d46958, 0x2537f0f, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:252 +0x43e
net/http.(*Client).send(0xc000bfaba0, 0xc249c4e900, 0x0, 0x0, 0x0, 0xc143d46958, 0x0, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).do(0xc000bfaba0, 0xc249c4e900, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:699 +0x44a
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:567
github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50
github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409
github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0556e3a40, 0xc0556e3aa0, 0xc0556e3b00, 0xc0ea676760, 0xc2079cb890, 0xc133c34c60, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4523809 [IO wait]:
internal/poll.runtime_pollWait(0x7f5992d88c90, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc21a409f18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc21a409f00, 0xc10ad9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc21a409f00, 0xc10ad9a000, 0x8000, 0x8000, 0x203078, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1c67f4230, 0xc10ad9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d25b8de0, 0xc249c7e118, 0x9, 0x9, 0xc0013d0700, 0x1007f63e742c368, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d25b8de0, 0xc249c7e118, 0x9, 0x9, 0x9, 0x601742e2, 0xc10d6e4dc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc249c7e118, 0x9, 0x9, 0x2b41140, 0xc1d25b8de0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc249c7e0e0, 0xc21589e0c0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc12e0d6300, 0xc236f20c30, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc12e0d6300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc12e0d6300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4496830 [semacquire]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00c51d1e0, 0xc18e916c18, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc2414a97d0, 0xc243035380, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1f8819380, 0xc214344370, 0x2, 0x2, 0x30, 0x0, 0xc003208000, 0xc0921ddae0, 0x30)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc1f8819380, 0xc23802c5a0, 0xc214344370, 0x2, 0x2, 0xc2016bb440, 0x2b44f80, 0xc2430354d0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc2430354a0, 0xc00b2e0e70)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 3720229 [select]:
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0x1, 0x1050101)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:391 +0x2ee
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).SyncBlocks.func1(0x2b99c80, 0xc000e9e080, 0xc0001e6a20, 0x20, 0x225c080)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:139 +0x3f
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks.func2(0xc1fc5dbfa0, 0xc008e14840, 0x26b38d8, 0x2b99c80, 0xc000e9e080, 0xc1fc5dbfb0, 0xc200f022e0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:186 +0x107
created by github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:182 +0x45d
goroutine 4462480 [semacquire]:
sync.runtime_Semacquire(0xc2430354b0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc2430354a8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc2430354a0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1e7c4fc50)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc232a72bd0, 0x255e040, 0x18, 0xc1ed8ff3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc2016bb440, 0x2bbccc0, 0xc1b5375c20, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc2016bb440, 0x7f63bf19f960, 0xc2016b9460, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc2016bb440, 0x2bbbd60, 0xc2016b9460, 0xc000863100, 0xc1ed187688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1b5375b80, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1b5375b60, 0xc1b53759e0, 0x26b3908, 0xc1b5375b60, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1b5375b60, 0x2b99d40, 0xc232a72a50)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375b40, 0xc1b53759e0, 0xc1b5375a00, 0x7f63e741f5d8, 0xc4eda5bc80)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375b40, 0xc1b53759e0, 0xc1b5375a00, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1b5375b40, 0x2b99d40, 0xc232a72a20)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0xc1b5375a20, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0x1, 0xc1b53759a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0xc1b5375a40, 0x2a324ece, 0xb57d4b885344)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0x20306d, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0xc1b5375a60, 0xc1b53759e0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc2271e5c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc18346acc0, 0xc1b53759e0, 0x26b3908, 0x2b99d40, 0xc232a729c0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469b00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469b00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0051527a0, 0xc0002c3380, 0x2bc6d40, 0xc0294cb380, 0xc183469b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4376735 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc1f17b26a0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1f17b2698)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1f17b2690, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc100498420)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1f17b25d0, 0x255e040, 0x18, 0xc101df13f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1f17c4000, 0x2bbccc0, 0xc0eba36820, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1f17c4000, 0x7f63bf19f960, 0xc00db9a250, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1f17c4000, 0x2bbbd60, 0xc00db9a250, 0xc000863100, 0xc101b69688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0eba36780, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0eba36760, 0xc0eba36600, 0x26b3908, 0xc0eba36760, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0eba36760, 0x2b99d40, 0xc1f17b2480)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0eba36740, 0xc0eba36600, 0xc0eba36620, 0x7f63e7428a30, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0eba36740, 0xc0eba36600, 0xc0eba36620, 0x2b99d01, 0xc0eba36740)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0eba36740, 0x2b99d40, 0xc1f17b2450)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0xc0eba36640, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0x7f599a973200, 0x20307c00000000)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0xc0eba36660, 0x1da2db7c, 0xb579c0e709fe)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0x20303a, 0xc0010000000000)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x7f59de527d00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0xc0eba36680, 0xc0eba36600, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc12884fc68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1f17c2000, 0xc0eba36600, 0x26b3908, 0x2b99d40, 0xc1f17b23f0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc09673ec00, 0xc1defd9700, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc09673ec00, 0xc1defd9700, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0eb34a3d0, 0xc0002c3380, 0x2bc6d40, 0xc09673ec00, 0xc1defd9700)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4514786 [IO wait]:
internal/poll.runtime_pollWait(0x7f5995457930, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281a80, 0xc233530000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281a80, 0xc233530000, 0x8000, 0x8000, 0x203062, 0x420bb5, 0xc188816820)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa438, 0xc233530000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc2442d67e0, 0xc0460c2ff8, 0x9, 0x9, 0xc00c5740a0, 0xc06edf4d60, 0x18)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc2442d67e0, 0xc0460c2ff8, 0x9, 0x9, 0x9, 0xc06edf4dc8, 0x40cfa8, 0xc02d3b8a60)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c2ff8, 0x9, 0x9, 0x2b41140, 0xc2442d67e0, 0x0, 0x0, 0x0, 0xc02d3b8a50)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2fc0, 0xc25c2e6330, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477e00, 0xc22f1db200, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582836 [runnable]:
internal/poll.runtime_pollWait(0x7f59e088d530, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00cd7b718, 0x72, 0xdb00, 0xdb99, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00cd7b700, 0xc248fda000, 0xdb99, 0xdb99, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00cd7b700, 0xc248fda000, 0xdb99, 0xdb99, 0x203094, 0x420bb5, 0xc25086b2e0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150ab4cf0, 0xc248fda000, 0xdb99, 0xdb99, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc25086b2e0, 0xc248fda000, 0xdb99, 0xdb99, 0x0, 0x41310b, 0xc02ad7a900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07a5073d8, 0x2b41380, 0xc25086b2e0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07a507180, 0x2b48520, 0xc150ab4cf0, 0x5, 0xc150ab4cf0, 0x40c924)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07a507180, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07a507180, 0xc23e39a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0fb8f86c0, 0xc23e39a000, 0x1000, 0x1000, 0xc0ca3477a0, 0xc02ad7ac58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc23e378780)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc23e378780, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc072dc6540)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0fb8f86c0)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4582669 [select]:
net/http.(*persistConn).writeLoop(0xc163269d40)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4463557 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc237c704c0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc237c704b8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc237c704b0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc2389e6000)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc237c703f0, 0x255e040, 0x18, 0xc1ef34b3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1ee23d4a0, 0x2bbccc0, 0xc0b013ea40, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1ee23d4a0, 0x7f63bf19f960, 0xc225d72e70, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1ee23d4a0, 0x2bbbd60, 0xc225d72e70, 0xc000863100, 0xc1ec633688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0b013e940, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0b013e920, 0xc0b013e760, 0x26b3908, 0xc0b013e920, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0b013e920, 0x2b99d40, 0xc237c702a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0b013e8e0, 0xc0b013e760, 0xc0b013e780, 0x7f63e743a180, 0xc4edda2180)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0b013e8e0, 0xc0b013e760, 0xc0b013e780, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0b013e8e0, 0x2b99d40, 0xc237c70270)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0xc0b013e7a0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0x1, 0xc0b013e720)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0xc0b013e7c0, 0x39027743, 0xb57b064c97c1)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0x20302c, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0xc0b013e7e0, 0xc0b013e760, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0f0ed7c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ee246480, 0xc0b013e760, 0x26b3908, 0x2b99d40, 0xc237c70210)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc00f65e300, 0xc1ee217400, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc00f65e300, 0xc1ee217400, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc019c06460, 0xc0002c3380, 0x2bc6d40, 0xc00f65e300, 0xc1ee217400)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4510187 [IO wait]:
internal/poll.runtime_pollWait(0x7f59943af680, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1ebaee018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1ebaee000, 0xc0cebc8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1ebaee000, 0xc0cebc8000, 0x8000, 0x8000, 0x203063, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004ffa020, 0xc0cebc8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc22a263d40, 0xc12d666498, 0x9, 0x9, 0xc0062d0300, 0x1007f63e743e848, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc22a263d40, 0xc12d666498, 0x9, 0x9, 0x9, 0x601742dc, 0xc22c8bddc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc12d666498, 0x9, 0x9, 0x2b41140, 0xc22a263d40, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc12d666460, 0xc08ed76900, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc19966bb00, 0xc23e2b1f80, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc19966bb00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc19966bb00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582636 [select]:
net/http.(*persistConn).writeLoop(0xc2386e6a20)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4084801 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc1aa0a9500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c
github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc000011360, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0xc29037f5c0, 0x40cfa8, 0x60, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77
github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9
github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc000bf82c0, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x0, 0x2000107, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b
github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc000c90ae0, 0x2b99c80, 0xc000e9e080, 0xc257835260, 0x25, 0x22dcf307, 0x287a7c5, 0x0, 0x7f59e0599060, 0x0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a
github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000270c00, 0x2b99c80, 0xc000e9e080, 0xc0c94f36e0, 0x20, 0x22dcf307, 0x287a7c5, 0x0, 0x0, 0x0, ...)
/__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc1a42187c0, 0x2b44ea0, 0xc18249caf0, 0xc20eba8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0x8e6caa1f6f17601, 0x9d4aa4ded66b726f, 0xc143f837c0, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x8e6caa1f6f17601, 0x9d4aa4ded66b726f, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0x8e6caa1f6f17601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc006837b00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4577213 [IO wait]:
internal/poll.runtime_pollWait(0x7f59c9f7d710, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06bec8a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06bec8a80, 0xc1b2e50000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06bec8a80, 0xc1b2e50000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7baca40)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc005f6b110, 0xc1b2e50000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc2608022a0, 0xc29067d0d8, 0x9, 0x9, 0xc001e05180, 0x1007f63e7434368, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc2608022a0, 0xc29067d0d8, 0x9, 0x9, 0x9, 0x601742ea, 0xc165f05dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc29067d0d8, 0x9, 0x9, 0x2b41140, 0xc2608022a0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc29067d0a0, 0xc1f810ba20, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc52a6fd200, 0xc1117f5a40, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc52a6fd200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc52a6fd200)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4465623 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc19785e2b8, 0x8, 0xc01a468960, 0x28, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc2228596c0, 0xc0b5eadfc0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc2228596c0, 0xc004ffa1c8, 0x1, 0x1, 0xaad796cb0b2e042, 0xf0a1fce18348224, 0x60aad9cb3a5ef05b, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc2228596c0, 0xc23b081a40, 0xc004ffa1c8, 0x1, 0x1, 0xc22276ec00, 0x2b44f80, 0xc216327fb0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc02c2a0d60, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc216327f80, 0xc222865d50)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582147 [semacquire]:
sync.runtime_SemacquireMutex(0xc007cd0344, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).IndexVersion(0xc007cd02c0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:141 +0x1f7
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0049f9000, 0xc084f17b90, 0x2, 0x2, 0x28bcc65066b49d01, 0x28784e46ada33c4d, 0x5a52b691f65db313, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1639 +0xfbf
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0040c6990, 0xc0049f9000, 0xc046eefd40, 0xc084f17b90, 0x2, 0x2, 0xc197b33380, 0x2b44f80, 0xc1e8866780, 0xc000cf0428, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1e8866750, 0xc1e8b40230)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4084491 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc134499b00, 0xc20e230000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc11c487320, 0xc20e230000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc173f54b70, 0xc20e230000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc128a48c30, 0x7f59e0599060, 0xc173f54b70, 0xc20e230000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc128a48c30, 0x7f59e0599060, 0xc173f54b70, 0xc20e230000, 0x8000, 0x8000, 0x2b6fc80, 0xc173f54b70, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0026ad240, 0x2b44ea0, 0xc128a48c30, 0xc20e230000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xf6f451aa76287701, 0x9dc833dc45eea49f, 0xc12deb9840, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xf6f451aa76287701, 0x9dc833dc45eea49f, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xf6f451aa76287701, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc0088b8c00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4514793 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02d3b9040, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0c0910c60, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1a3430180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4561860 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc28c43a870, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc079aead80, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0ab2bbe00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4582114 [select]:
net/http.(*persistConn).writeLoop(0xc2433439e0)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4508842 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc4b81e0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4561339 [IO wait]:
internal/poll.runtime_pollWait(0x7f5884b38c40, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc23562ac98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc23562ac80, 0xc1debe4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc23562ac80, 0xc1debe4000, 0x8000, 0x8000, 0x400000801, 0xc100000001, 0x8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1505a4570, 0xc1debe4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d8a901e0, 0xc0b275ba78, 0x9, 0x9, 0xc196e8cae0, 0xc227e27df7, 0xc227e27d80)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d8a901e0, 0xc0b275ba78, 0x9, 0x9, 0x9, 0x406594, 0xc173cee660, 0xc227e27df7)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0b275ba78, 0x9, 0x9, 0x2b41140, 0xc1d8a901e0, 0x0, 0xc200000000, 0x2124f40, 0xc1aa1bb680)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0b275ba40, 0xc1aa1bb660, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc136f0c900, 0xc290853aa0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc136f0c900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc136f0c900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582672 [select]:
net/http.(*persistConn).roundTrip(0xc0fb8f86c0, 0xc1e07df000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2559 +0x763
net/http.(*Transport).roundTrip(0xc000ce5540, 0xc05bf41200, 0xc28e8c06c0, 0x40f967, 0xc000069538)
/usr/local/go/src/net/http/transport.go:582 +0xa88
net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc05bf41200, 0xc000ce5540, 0x0, 0x0)
/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc05bf41200, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc9e3ffeec8, 0x2537f0f, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:252 +0x43e
net/http.(*Client).send(0xc000bfaba0, 0xc05bf41200, 0x0, 0x0, 0x0, 0xc9e3ffeec8, 0x0, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).do(0xc000bfaba0, 0xc05bf41200, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:699 +0x44a
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:567
github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50
github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409
github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc07f961a40, 0xc07f961b00, 0xc07f961bc0, 0xc00926ed90, 0xc225a73c80, 0xc044daa9a0, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4514692 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c0239548, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281980, 0xc22f1f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281980, 0xc22f1f6000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000067)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa428, 0xc22f1f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc23b73b680, 0xc0460c2d58, 0x9, 0x9, 0xc2293b7d40, 0x457243, 0xb5b9d887065e)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc23b73b680, 0xc0460c2d58, 0x9, 0x9, 0x9, 0xbffdee9a6df86104, 0x15c1621decc, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c2d58, 0x9, 0x9, 0x2b41140, 0xc23b73b680, 0x0, 0x0, 0xc25e600ce0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c2d20, 0xc25e600ce0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21d477500, 0xc22f1da870, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc21d477500)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc21d477500)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4582522 [select]:
net/http.(*persistConn).readLoop(0xc9aa80f7a0)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4561338 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc136f0c900)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4514795 [IO wait]:
internal/poll.runtime_pollWait(0x7f58853b1910, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc238281b98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc238281b80, 0xc233578000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc238281b80, 0xc233578000, 0x8000, 0x8000, 0x203067, 0x420bb5, 0xc19c662500)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0060fa458, 0xc233578000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0c0910420, 0xc0460c3298, 0x9, 0x9, 0xc0072d2e00, 0x1007f63e7434a30, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc0c0910420, 0xc0460c3298, 0x9, 0x9, 0x9, 0x601742e9, 0xc0594e9dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0460c3298, 0x9, 0x9, 0x2b41140, 0xc0c0910420, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0460c3260, 0xc25cb2f500, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1a3430180, 0xc22f1db830, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1a3430180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1a3430180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4084495 [chan receive]:
github.com/minio/minio-go/v7.(*Object).doGetRequest(...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276
github.com/minio/minio-go/v7.(*Object).Read(0xc2230af680, 0xc23e980000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227
github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc23e8fa660, 0xc23e980000, 0x8000, 0x8000, 0x1, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55
github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc225a73bf0, 0xc23e980000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58
io.copyBuffer(0x2b44ea0, 0xc02ebddc70, 0x7f59e0599060, 0xc225a73bf0, 0xc23e980000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f59e0599060)
/usr/local/go/src/io/io.go:405 +0x122
io.CopyBuffer(0x2b44ea0, 0xc02ebddc70, 0x7f59e0599060, 0xc225a73bf0, 0xc23e980000, 0x8000, 0x8000, 0x2b6fc80, 0xc225a73bf0, 0x0)
/usr/local/go/src/io/io.go:378 +0x82
github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc00320edc0, 0x2b44ea0, 0xc02ebddc70, 0xc23e980000, 0x8000, 0x8000, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4
github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000e9e080, 0x7f63bf037140, 0xc000270c00, 0xfe73c0b287ee7601, 0xfb22083c60000713, 0xc11632f3c0, 0x37, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300
github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfe73c0b287ee7601, 0xfb22083c60000713, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e
github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000b25680, 0x2b99c80, 0xc000e9e080, 0x2b42d60, 0xc000681710, 0x7f63bf037140, 0xc000270c00, 0xc000592940, 0xf, 0xfe73c0b287ee7601, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c
github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc007031700, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7
github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d1d1faa0, 0xc0001e6a20, 0x2b99c80, 0xc000e9e080, 0xc159ebf8e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61
created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a
goroutine 4582293 [runnable]:
internal/poll.runtime_pollWait(0x7f5886377210, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1e8864d18, 0x72, 0xc900, 0xc92b, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1e8864d00, 0xc242c92000, 0xc92b, 0xc92b, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1e8864d00, 0xc242c92000, 0xc92b, 0xc92b, 0x203095, 0x420bb5, 0xc255812920)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc1c67f5108, 0xc242c92000, 0xc92b, 0xc92b, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc255812920, 0xc242c92000, 0xc92b, 0xc92b, 0x410, 0x44c7, 0xc24645ee90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07a983ad8, 0x2b41380, 0xc255812920, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07a983880, 0x2b48520, 0xc1c67f5108, 0x5, 0xc1c67f5108, 0x400)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07a983880, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07a983880, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0f7ded7a0, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc23e379380, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc23c57ba80, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc997c7fac0, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x400, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc997c7fac0, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc997c7fb00, 0xc2270dc5aa, 0x5a56, 0x5a56, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc997c7fb00, 0xc2270da000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1db5cf1a0, 0xc1db5cf200, 0xc1db5cf260, 0xc0c5f020b0, 0xc1a1b7c570, 0xc13f938160, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4582751 [select]:
net/http.(*persistConn).writeLoop(0xc10609c000)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4582743 [select]:
net/http.(*persistConn).writeLoop(0xc24422a900)
/usr/local/go/src/net/http/transport.go:2336 +0x11c
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1707 +0xc7b
goroutine 4556671 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc183057e00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4463649 [semacquire, 3 minutes]:
sync.runtime_Semacquire(0xc216327f90)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc216327f88)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc216327f80, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc23b0830b0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc216327ec0, 0x255e040, 0x18, 0xc1e5d6d3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc22276ec00, 0x2bbccc0, 0xc150601c00, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc22276ec00, 0x7f63bf19f960, 0xc1fb5f97c0, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc22276ec00, 0x2bbbd60, 0xc1fb5f97c0, 0xc000863100, 0xc0fe16d688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc150601b60, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc150601b40, 0xc1506019e0, 0x26b3908, 0xc150601b40, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc150601b40, 0x2b99d40, 0xc216327d70)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc150601b20, 0xc1506019e0, 0xc150601a00, 0x7f63e743af10, 0xc10000cd80)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc150601b20, 0xc1506019e0, 0xc150601a00, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc150601b20, 0x2b99d40, 0xc216327d40)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0xc150601a20, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0x1, 0xc1506019a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0xc150601a40, 0x6517b34, 0xb57ad39b9bad)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0x203054, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0xc150601a60, 0xc1506019e0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc102211c68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1ef9c49c0, 0xc1506019e0, 0x26b3908, 0x2b99d40, 0xc216327ce0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc071fc8780, 0xc1efed1a00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc071fc8780, 0xc1efed1a00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc075a62c50, 0xc0002c3380, 0x2bc6d40, 0xc071fc8780, 0xc1efed1a00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4561273 [IO wait]:
internal/poll.runtime_pollWait(0x7f59decac5a0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc2900c3098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc2900c3080, 0xc13121c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc2900c3080, 0xc13121c000, 0x1000, 0x1000, 0x4378cc, 0xc15e695b58, 0x4647d0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc007b7e060, 0xc13121c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
net/http.(*persistConn).Read(0xc27abf85a0, 0xc13121c000, 0x1000, 0x1000, 0xc176cdc300, 0xc15e695c58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc1ee584c60)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc1ee584c60, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc169b849c0)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc27abf85a0)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4582100 [IO wait]:
internal/poll.runtime_pollWait(0x7f5886af6a40, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc19c7d8f98, 0x72, 0xd700, 0xd761, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc19c7d8f80, 0xc2520a8000, 0xd761, 0xd761, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc19c7d8f80, 0xc2520a8000, 0xd761, 0xd761, 0x203095, 0x420bb5, 0xc2552394e0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc9cbac3438, 0xc2520a8000, 0xd761, 0xd761, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc2552394e0, 0xc2520a8000, 0xd761, 0xd761, 0x2070, 0x4ee0, 0xc184f9ae90)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07aca3758, 0x2b41380, 0xc2552394e0, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07aca3500, 0x2b48520, 0xc9cbac3438, 0x5, 0xc9cbac3438, 0x2060)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07aca3500, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07aca3500, 0xc23e98089a, 0x7766, 0x7766, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc22ca94a20, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).Read(0xc23e27ee40, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:212 +0x141
io.(*LimitedReader).Read(0xc250f160c0, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0)
/usr/local/go/src/io/io.go:451 +0x63
net/http.(*body).readLocked(0xc003233400, 0xc23e98089a, 0x7766, 0x7766, 0x89a, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:850 +0x5f
net/http.(*body).Read(0xc003233400, 0xc23e98089a, 0x7766, 0x7766, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:842 +0xf2
net/http.(*bodyEOFSignal).Read(0xc003233440, 0xc23e98089a, 0x7766, 0x7766, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2710 +0xde
io.ReadAtLeast(0x2b48600, 0xc003233440, 0xc23e980000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1e1e756e0, 0xc1e1e75740, 0xc1e1e757a0, 0xc16fc4fd10, 0xc1970f7e00, 0xc083acd550, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4463565 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00c51d1e0, 0xc0eba24bf8, 0x8, 0xc1a3b90f00, 0x28, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:154 +0x28c
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1ee227d00, 0xc0ccb232a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1ee227d00, 0xc151b900e8, 0x1, 0x1, 0xc1be9f2950, 0xd, 0x1, 0xc00832e380, 0x7f63e743ca30)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc1ee227d00, 0xc1ee23dbc0, 0xc151b900e8, 0x1, 0x1, 0xc1ee23d4a0, 0x2b44f80, 0xc237c704e0, 0xc000ce6120, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc0874b4ac0, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc237c704b0, 0xc007134d90)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582730 [select]:
net/http.(*persistConn).roundTrip(0xc0fc7806c0, 0xc0052cf040, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:2559 +0x763
net/http.(*Transport).roundTrip(0xc000ce5540, 0xc0614d7600, 0xc1ef8026c0, 0x40f967, 0xc000142e58)
/usr/local/go/src/net/http/transport.go:582 +0xa88
net/http.(*Transport).RoundTrip(0xc000ce5540, 0xc0614d7600, 0xc000ce5540, 0x0, 0x0)
/usr/local/go/src/net/http/roundtrip.go:17 +0x35
net/http.send(0xc0614d7600, 0x2b485c0, 0xc000ce5540, 0x0, 0x0, 0x0, 0xc0071fa9f0, 0x2537f0f, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:252 +0x43e
net/http.(*Client).send(0xc000bfaba0, 0xc0614d7600, 0x0, 0x0, 0x0, 0xc0071fa9f0, 0x0, 0x1, 0x0)
/usr/local/go/src/net/http/client.go:176 +0xfa
net/http.(*Client).do(0xc000bfaba0, 0xc0614d7600, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:699 +0x44a
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:567
github.com/minio/minio-go/v7.Client.do(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50
github.com/minio/minio-go/v7.Client.executeMethod(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409
github.com/minio/minio-go/v7.Client.getObject(0xc0002f3600, 0xc000bb0600, 0x1, 0xc00093c620, 0x14, 0xc0007de250, 0x10, 0x1, 0xc000bfaba0, 0xc000b8e8e0, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b
github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0cc8dcea0, 0xc0cc8dcf00, 0xc0cc8dd0e0, 0xc1d3fe4c40, 0xc24bf2e1b0, 0xc0a306e420, 0x2b99c80, 0xc000e9e080, 0x7ffee677808b, 0x1a, ...)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d
created by github.com/minio/minio-go/v7.Client.GetObject
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346
goroutine 4582710 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c032e7c0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1a0ead718, 0x72, 0x1000, 0x109b, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1a0ead700, 0xc14f802000, 0x109b, 0x109b, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1a0ead700, 0xc14f802000, 0x109b, 0x109b, 0x203094, 0x420bb5, 0xc2517d6700)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc150ab4cd0, 0xc14f802000, 0x109b, 0x109b, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc2517d6700, 0xc14f802000, 0x109b, 0x109b, 0x0, 0x41310b, 0xc153e88900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc07a506cd8, 0x2b41380, 0xc2517d6700, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc07a506a80, 0x2b48520, 0xc150ab4cd0, 0x5, 0xc150ab4cd0, 0x463b40)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07a506a80, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc07a506a80, 0xc19df3f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0f41e47e0, 0xc19df3f000, 0x1000, 0x1000, 0xc06ed7aa80, 0xc153e88c58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc1561881e0)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc1561881e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc25d3c2120)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0f41e47e0)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4562107 [IO wait]:
internal/poll.runtime_pollWait(0x7f5886856eb8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc28f145698, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc28f145680, 0xc240ed7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc28f145680, 0xc240ed7000, 0x1000, 0x1000, 0x4db33a, 0xb5be9e8c5b7d, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc007b7e5c0, 0xc240ed7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
net/http.(*connReader).Read(0xc272ee0900, 0xc240ed7000, 0x1000, 0x1000, 0xbffdeeb69dc65277, 0x1765542fa10, 0x3f6bae0)
/usr/local/go/src/net/http/server.go:797 +0xf4
bufio.(*Reader).fill(0xc2234f2cc0)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc2234f2cc0, 0x4, 0x1765542fa10, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc285589360, 0x2b99c80, 0xc00d5b9980)
/usr/local/go/src/net/http/server.go:1949 +0x9d7
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2962 +0x35c
goroutine 4582520 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a90be4008, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc23ac1a898, 0x72, 0xfb00, 0xfb29, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc23ac1a880, 0xc25b894000, 0xfb29, 0xfb29, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc23ac1a880, 0xc25b894000, 0xfb29, 0xfb29, 0x203095, 0x420bb5, 0xc256c4d260)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xca27c505d0, 0xc25b894000, 0xfb29, 0xfb29, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc256c4d260, 0xc25b894000, 0xfb29, 0xfb29, 0x0, 0x41310b, 0xc05545b900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc073b25058, 0x2b41380, 0xc256c4d260, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc073b24e00, 0x2b48520, 0xca27c505d0, 0x5, 0xca27c505d0, 0x40c924)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc073b24e00, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc073b24e00, 0xc23b142000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc0fd6c8000, 0xc23b142000, 0x1000, 0x1000, 0xc0cc95a720, 0xc05545bc58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc147ddb560)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc147ddb560, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0167b3560)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0fd6c8000)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4464097 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc238b3e430)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc238b3e428)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc238b3e420, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc238b582d0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc238b3e360, 0x255e040, 0x18, 0xc1ea8313f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc23856be00, 0x2bbccc0, 0xc0ccb1ba80, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc23856be00, 0x7f63bf19f960, 0xc238a7a690, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc23856be00, 0x2bbbd60, 0xc238a7a690, 0xc000863100, 0xc0062ed688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc0ccb1b9e0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0ccb1b9c0, 0xc0ccb1b860, 0x26b3908, 0xc0ccb1b9c0, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc0ccb1b9c0, 0x2b99d40, 0xc238b3e210)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0ccb1b9a0, 0xc0ccb1b860, 0xc0ccb1b880, 0x7f63e7430a30, 0xc040da4180)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0ccb1b9a0, 0xc0ccb1b860, 0xc0ccb1b880, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc0ccb1b9a0, 0x2b99d40, 0xc238b3e1e0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0xc0ccb1b8a0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0x1, 0xc0ccb1b820)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0xc0ccb1b8c0, 0x3b240f32, 0xb57b086e2fab)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0x203033, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0xc0ccb1b8e0, 0xc0ccb1b860, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc2356cac68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc21c6b8900, 0xc0ccb1b860, 0x26b3908, 0x2b99d40, 0xc238b3e180)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00, 0xc23857f000, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc023e1ac00, 0xc23857f000, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc03f97d970, 0xc0002c3380, 0x2bc6d40, 0xc023e1ac00, 0xc23857f000)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4582742 [select]:
net/http.(*persistConn).readLoop(0xc24422a900)
/usr/local/go/src/net/http/transport.go:2158 +0x9a0
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4561861 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0ab2bbe00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4581972 [semacquire]:
sync.runtime_SemacquireMutex(0xc00ddea134, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00ddea0b0, 0xc0d278c2c8, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc0db4967d0, 0xc114311bf0, 0x253e210, 0xb, 0xc1e914a0e0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1693 +0x6f0
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc99c97fdc0, 0xc07df928b0, 0x2, 0x2, 0xc1683f3120, 0xd, 0x1, 0xc222d2d500, 0x7f63e742a180)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00618cb10, 0xc99c97fdc0, 0xc121ce5f80, 0xc07df928b0, 0x2, 0x2, 0xc121ce5920, 0x2b44f80, 0xc114311cb0, 0x3f6bae0, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc114311c80, 0xc1ea0c61c0)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4498125 [semacquire]:
sync.runtime_SemacquireMutex(0xc00c51d264, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).RLock(...)
/usr/local/go/src/sync/rwmutex.go:50
github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00c51d1e0, 0xc18980ebfd, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:180 +0x294
github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc2442bb7d0, 0xc122210780, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403
github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc23ea43ec0, 0xc23887b2f0, 0x2, 0x2, 0xc0fb0fa610, 0xd, 0x1, 0xc005ff4000, 0x7f63e740ae98)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7
github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00346a750, 0xc23ea43ec0, 0xc1d675ecc0, 0xc23887b2f0, 0x2, 0x2, 0xc1549fe360, 0x2b44f80, 0xc1222108d0, 0xc00017c0b8, ...)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1222108a0, 0xc233170380)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4513293 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc065466f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4474341 [select, 3 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1918b0600)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582731 [select]:
github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0cc8dd140, 0xa, 0x2b99c80, 0xc0052cef00, 0xc24bf2e240)
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c
created by github.com/minio/minio-go/v7.Client.newRetryTimer
/__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163
goroutine 4514540 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc062f5e320, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc12b314900, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc054d93b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4510178 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc19966b800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4578950 [IO wait]:
internal/poll.runtime_pollWait(0x7f59deb9daa0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc53c8d2318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc53c8d2300, 0xc1ac984000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc53c8d2300, 0xc1ac984000, 0x8000, 0x8000, 0x203079, 0x420bb5, 0xc1e7194c20)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00450cda0, 0xc1ac984000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1ff86f1a0, 0xc125fe7d18, 0x9, 0x9, 0xc00596ce00, 0x1007f63e741ef10, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1ff86f1a0, 0xc125fe7d18, 0x9, 0x9, 0x9, 0x601742e9, 0xc036818dc8, 0x2ade866)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc125fe7d18, 0x9, 0x9, 0x2b41140, 0xc1ff86f1a0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc125fe7ce0, 0xc258491de0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1e65f3800, 0xc19a278840, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1e65f3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1e65f3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4515934 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00bc80f00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4515945 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc028eec180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4577951 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0c41d3800)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4513604 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc10a44af00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4515928 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc07661ac30, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19149ea80, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00bc80d80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4388078 [semacquire, 4 minutes]:
sync.runtime_Semacquire(0xc0f4f5bfc0)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc0f4f5bfb8)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc0f4f5bfb0, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0ff209d40)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0f4f5bef0, 0x255e040, 0x18, 0xc0f55533f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc1c9270d80, 0x2bbccc0, 0xc088f91540, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc1c9270d80, 0x7f63bf19f960, 0xc0fb69f2d0, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc1c9270d80, 0x2bbbd60, 0xc0fb69f2d0, 0xc000863100, 0xc005b4d688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc088f914a0, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc088f91480, 0xc088f91320, 0x26b3908, 0xc088f91480, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc088f91480, 0x2b99d40, 0xc0f4f5bda0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc088f91460, 0xc088f91320, 0xc088f91340, 0x7f63e740c2f0, 0x0)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc088f91460, 0xc088f91320, 0xc088f91340, 0x2b99d01, 0xc088f91460)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc088f91460, 0x2b99d40, 0xc0f4f5bd70)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0xc088f91360, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0x1, 0xc088f912e0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0xc088f91380, 0x1d73b811, 0xb579c0b7e68f)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0x203022, 0x8701ba)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x40c700, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0xc088f913a0, 0xc088f91320, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc11e6edc68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc0f4f4a0c0, 0xc088f91320, 0x26b3908, 0x2b99d40, 0xc0f4f5bd10)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780, 0xc0fb6a6a00, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc04ffaa780, 0xc0fb6a6a00, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0ca383b40, 0xc0002c3380, 0x2bc6d40, 0xc04ffaa780, 0xc0fb6a6a00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4517272 [IO wait]:
internal/poll.runtime_pollWait(0x7f58871547d0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc06ce7e398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc06ce7e380, 0xc1495b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc06ce7e380, 0xc1495b2000, 0x8000, 0x8000, 0x203086, 0x0, 0x60100000008)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc143d460a8, 0xc1495b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1f71baba0, 0xc0a65ff538, 0x9, 0x9, 0xc1ee092a80, 0x1007f63e7424368, 0x0)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1f71baba0, 0xc0a65ff538, 0x9, 0x9, 0x9, 0x601742e2, 0xc071598dc8, 0x4db7f6)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0a65ff538, 0x9, 0x9, 0x2b41140, 0xc1f71baba0, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a65ff500, 0xc0a8b0c0a0, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc003a9ea80, 0xc246e60a50, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc003a9ea80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc003a9ea80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4556749 [running]:
goroutine running on other thread; stack unavailable
created by golang.org/x/sync/errgroup.(*Group).Go
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66
goroutine 4582071 [semacquire]:
sync.runtime_Semacquire(0xc1e8866760)
/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1e8866758)
/usr/local/go/src/sync/waitgroup.go:130 +0x64
golang.org/x/sync/errgroup.(*Group).Wait(0xc1e8866750, 0x0, 0x2b4a640)
/__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc19d4c7fb0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33
github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1e8866660, 0x255e040, 0x18, 0xc16079d3f8, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc
github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0001e6a20, 0xc197b33380, 0x2bbccc0, 0xc1e886a600, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3
github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001ce000, 0xc197b33380, 0x7f63bf19f960, 0xc009911860, 0x0, 0x0)
/__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da
github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc000863100, 0xc197b33380, 0x2bbbd60, 0xc009911860, 0xc000863100, 0xc2400cd688)
/__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82
github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc000863100, 0x2bb8400, 0xc1e886a560, 0x24321a0, 0xc00046a000)
/__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109
github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e886a540, 0xc1e886a3e0, 0x26b3908, 0xc1e886a540, 0x23b6440)
/__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb8700, 0xc1e886a540, 0x2b99d40, 0xc1e8866510)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e886a520, 0xc1e886a3e0, 0xc1e886a400, 0x7f63e742f5d8, 0xc1eb50be00)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e886a520, 0xc1e886a3e0, 0xc1e886a400, 0x2ade86a, 0x0)
/__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb69c0, 0xc1e886a520, 0x2b99d40, 0xc1e88664e0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0xc1e886a420, 0x0, 0x0)
/__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0x1, 0xc1e886a3a0)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0xc1e886a440, 0xa9fde99, 0xb5ba67fb6914)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0x20307a, 0x420bb5)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc000385cd0, 0x420b00, 0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0xc1e886a460, 0xc1e886a3e0, ...)
/__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc0005ecc68, 0x40cfa8)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc000863100, 0x2bb7020, 0xc1dc394840, 0xc1e886a3e0, 0x26b3908, 0x2b99d40, 0xc1e8866480)
/__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00, 0xc0e250f300, 0xc000844d80, 0x3f3f8a0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc0002c3380, 0x2bc6d40, 0xc1038bcf00, 0xc0e250f300, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc10888b490, 0xc0002c3380, 0x2bc6d40, 0xc1038bcf00, 0xc0e250f300)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1
goroutine 4486230 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0e90c9540, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc12843aea0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1c1c82180)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4511257 [IO wait]:
internal/poll.runtime_pollWait(0x7f59dae18fc8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1951f1918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1951f1900, 0xc1ffc60000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1951f1900, 0xc1ffc60000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000073)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc9e49f4068, 0xc1ffc60000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc1d891e540, 0xc0a63f4f18, 0x9, 0x9, 0xc227e09d40, 0x457243, 0xb5b93aff39c9)
/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x2b41140, 0xc1d891e540, 0xc0a63f4f18, 0x9, 0x9, 0x9, 0xbffdee99c7a47217, 0x15b78985b71, 0x3f6bae0)
/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0a63f4f18, 0x9, 0x9, 0x2b41140, 0xc1d891e540, 0x0, 0x0, 0xc1fe98ae60, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a63f4ee0, 0xc1fe98ae60, 0x3f6bae0, 0x0, 0x0)
/__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1245db680, 0xc10e4704e0, 0x26b66b0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc0002c3380, 0x2bc6d40, 0xc1245db680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0002c3380, 0x2bc6d40, 0xc1245db680)
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
/__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd
goroutine 4507827 [IO wait]:
internal/poll.runtime_pollWait(0x7f5889b51af0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc07c6dea98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc07c6dea80, 0xc14940f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc07c6dea80, 0xc14940f000, 0x1000, 0x1000, 0x4db33a, 0xb5bf2411151e, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000714048, 0xc14940f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
net/http.(*connReader).Read(0xc003542180, 0xc14940f000, 0x1000, 0x1000, 0xbffdeeb78bcb3a74, 0x17731b309f5, 0x3f6bae0)
/usr/local/go/src/net/http/server.go:797 +0xf4
bufio.(*Reader).fill(0xc1d73b00c0)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc1d73b00c0, 0x4, 0x17731b309f5, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc0acf443c0, 0x2b99c80, 0xc233135f00)
/usr/local/go/src/net/http/server.go:1949 +0x9d7
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2962 +0x35c
goroutine 4578955 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0fe80c140, 0x1, 0x0, 0x0, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc19958b140, 0x0, 0x0)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc079945980)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1
goroutine 4517271 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc003a9ea80)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
goroutine 4582113 [IO wait]:
internal/poll.runtime_pollWait(0x7f63c07862c0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc1fffd2398, 0x72, 0xba00, 0xba45, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1fffd2380, 0xc20a024000, 0xba45, 0xba45, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc1fffd2380, 0xc20a024000, 0xba45, 0xba45, 0x203095, 0x420bb5, 0xc256620180)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00a33bfd0, 0xc20a024000, 0xba45, 0xba45, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x8e
crypto/tls.(*atLeastReader).Read(0xc256620180, 0xc20a024000, 0xba45, 0xba45, 0x0, 0x41310b, 0xc13be0b900)
/usr/local/go/src/crypto/tls/conn.go:760 +0x60
bytes.(*Buffer).ReadFrom(0xc22c4ed758, 0x2b41380, 0xc256620180, 0x40a685, 0x21a56c0, 0x2489b40)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
crypto/tls.(*Conn).readFromUntil(0xc22c4ed500, 0x2b48520, 0xc00a33bfd0, 0x5, 0xc00a33bfd0, 0xc08f6a2900)
/usr/local/go/src/crypto/tls/conn.go:782 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc22c4ed500, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:589 +0x115
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:557
crypto/tls.(*Conn).Read(0xc22c4ed500, 0xc2007ca000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1233 +0x15b
net/http.(*persistConn).Read(0xc2433439e0, 0xc2007ca000, 0x1000, 0x1000, 0xc1647adce0, 0xc13be0bc58, 0x405125)
/usr/local/go/src/net/http/transport.go:1884 +0x75
bufio.(*Reader).fill(0xc1977abce0)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc1977abce0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0b5fbbb60)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc2433439e0)
/usr/local/go/src/net/http/transport.go:2037 +0x1a8
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1706 +0xc56
goroutine 4514699 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21d477b00)
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment