Created
February 1, 2021 04:24
-
-
Save amckinley/48a4f53a064362eae912e757ab52a1cf to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| level=warn ts=2021-01-31T23:52:39.276157883Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6WSYJWHRKSQKBFC9QAVVZT:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276323918Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6NXS9NWZ3BX9029Z3N2SVG:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276342535Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6NXWQXM8AZQ7K8KTG6B63D:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276160506Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6NY6TP74K6RGH9RKH1FVEM:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276513391Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6WV1QPZB49F281T8WN0QVA:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276529324Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6WT228JD2B47QCSAD8EVWJ:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276554065Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6WSVJ7AP4123R29950SRAF:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.27659266Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6NYYZV9BGPKS0GJ3JSJZC2:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:39.276603992Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6NZFFWWJC4ZQ4QJVCSBJ4V:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=info ts=2021-01-31T23:52:41.678866476Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.94002167s id=01EVK60YHBP0VKDV0SEMZNAYJN | |
| level=info ts=2021-01-31T23:52:41.68009796Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.585145822s id=01EW3FVR54BF769E8ZZJMB74Y0 | |
| level=info ts=2021-01-31T23:52:41.681245678Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.957290138s id=01ET7D8A53PYYAPV34VGEH02EG | |
| level=info ts=2021-01-31T23:52:41.682356545Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.584426628s id=01ETQG6REHFJDE1RJNZN4Q5F2E | |
| level=info ts=2021-01-31T23:52:41.68342913Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.575654087s id=01EV7K7E2TJHXE1JQTSHNCQT3V | |
| level=info ts=2021-01-31T23:52:41.684515812Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.946730967s id=01EVTPAVRHACWJSH2VNW3JFNS7 | |
| level=info ts=2021-01-31T23:52:41.697155772Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.59145999s id=01EX7HCQ5NCDGG3VT9NM5WMR72 | |
| level=info ts=2021-01-31T23:52:41.713503553Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.500455481s id=01ERT4EBTJB9FWARF034ZSEJ5E | |
| level=info ts=2021-01-31T23:52:41.714627011Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.545073077s id=01EX7Z59QCHF56XV6K1S0DCQR8 | |
| level=warn ts=2021-01-31T23:52:43.833333144Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EXBKVRNTX98RMPJ08EMRJN5G:Qfb5KQdJUfWia8Vrii7sBkXPIZmMzc9vnKYSx_HfsEk err="read tcp 10.11.247.204:57306->10.11.224.48:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:43.83343362Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EXBTRC7MPXF682ATY9JE31QR:Qfb5KQdJUfWia8Vrii7sBkXPIZmMzc9vnKYSx_HfsEk err="read tcp 10.11.247.204:57330->10.11.224.48:11211: i/o timeout" | |
| level=info ts=2021-01-31T23:52:45.084100221Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.303336017s id=01ET9ZMSD9CAM1H5CN3PHVHBCX | |
| level=info ts=2021-01-31T23:52:45.085378057Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.30861568s id=01EW6FZGJW04ZS4GRF629ES1FG | |
| level=info ts=2021-01-31T23:52:45.102665956Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.321867124s id=01ETMG43R4EH7AZTTNH52SNS8R | |
| level=info ts=2021-01-31T23:52:45.105695027Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.335997353s id=01EW2DGH55KQ6BY96E4VRQC4MC | |
| level=info ts=2021-01-31T23:52:45.106897966Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.326126593s id=01ET8X9W4F5Z1AJ6C27ZAG4S25 | |
| level=info ts=2021-01-31T23:52:45.108020168Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.335470757s id=01ETK6X2S9WTHEH0CGNTZ5XT91 | |
| level=info ts=2021-01-31T23:52:45.109300794Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=3.412117024s id=01EWVQR0YDK0T25R9F2XV2HM2W | |
| level=info ts=2021-01-31T23:52:45.111875471Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.329770634s id=01ET1KW5R54KNTEV0M235YYW7M | |
| level=info ts=2021-01-31T23:52:45.113038954Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.340485716s id=01EVKKQ2PMV86W44SN33ZS0F5D | |
| level=info ts=2021-01-31T23:52:45.114148911Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.340137929s id=01EVKTKECCY6772ND4GAYHB6G4 | |
| level=info ts=2021-01-31T23:52:45.120967769Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.360332633s id=01EVW6CDN779N899ASGV1FXGVK | |
| level=info ts=2021-01-31T23:52:45.122065935Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.345317929s id=01EX316XR7ZWFEBHQQ7KG2SYT9 | |
| level=warn ts=2021-01-31T23:52:45.292328054Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3172 firstKey=S:01EXC8FBSGADA327QZZG83QSVJ:288012448 err="read tcp 10.11.247.204:32848->10.11.105.37:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:45.292579892Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3038 firstKey=S:01EXC8G6KYK22JH23N91P5VHXY:271262944 err="read tcp 10.11.247.204:32944->10.11.105.37:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:45.29267056Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3227 firstKey=S:01EXC8FVGEA3V1DM2AB3DNM4G6:284565152 err="read tcp 10.11.247.204:32856->10.11.105.37:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:45.292710986Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3204 firstKey=S:01EXC8G60TNT35BDACM9N250Y6:285506320 err="read tcp 10.11.247.204:60876->10.11.105.37:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:45.293522574Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3161 firstKey=S:01EXC8F0C7J4N7CQNXFEA0QKC3:283130288 err="read tcp 10.11.247.204:60900->10.11.105.37:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:45.30210219Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3093 firstKey=S:01EXC8EQRRK48YMDZ50DR8F2ZF:280491152 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:45.351271666Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=9.20384703s err="rpc error: code = Aborted desc = fetch series for block 01EWVGW6Z0ABX7Y7TH93TH3Q5C: preload chunks: read range for 2: get range reader: failed to get object attributes: fake/01EWVGW6Z0ABX7Y7TH93TH3Q5C/chunks/000003: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:52:45.351289134Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EWVGWWRXD9JE0068KDHJT1C5:LwNKvQteHuEsb9i-lyH5jCSeicCE04zRbCQKWAp1xpw err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:45.351864562Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=8.514136574s err="rpc error: code = Aborted desc = fetch series for block 01EWVGW6Z0ABX7Y7TH93TH3Q5C: preload chunks: read range for 2: get range reader: failed to get object attributes: fake/01EWVGW6Z0ABX7Y7TH93TH3Q5C/chunks/000003: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:52:47.502500614Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX0ESXYMY6W59Z2A5FF7B1TZ:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:47.502728974Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX55WN98HNMVDSTD32BQ1711:kTq5ljo8-NzirzrhAY0zUmSPlbfacS39Zd6S5dwAX9k err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:47.881866638Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EXBTR3X6VY51RNHVXTK5QHQ3:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:47.881896943Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX0NP3TE1QB12N7GKD4QH33M:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.374536354Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EXAZ9BEBF41H61MRVK6JM341:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.374664267Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX7R9K74CHAF7FR12JHD1PHM:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.375969831Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EWXEP6RFX4DGC2PN7M2KRSQE:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:48.376405803Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX2TC9HDDZPHHZBWN4MZQSZF:HdiP7OEss96g6SMf6T_o_gwCJHO3y_HMt1Vsd0Dcd9g err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:48.378372903Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EWVGX1N0RDW6937SFFN95PAW:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.646817836Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EXAZ8WJ5RTDQTXPT5DKWZ9M0:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.657836662Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EWY3A827D2GX5HX1WH11Y4VZ:HdiP7OEss96g6SMf6T_o_gwCJHO3y_HMt1Vsd0Dcd9g err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.664596566Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX8CVN38WW6FJY9AS9V9KZ1E:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:48.732390848Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX0NP3TE1QB12N7GKD4QH33M:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:48.760629006Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX8KS3NDFYGPC3SRWXEC96HM:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:48.774286604Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EXAZA35KWY775ZH9SA59FES4:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:48.972078329Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EXAZ9KQRQ5HGS8N03BQGREYX:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:49.077152445Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=P:01EWY3A827D2GX5HX1WH11Y4VZ:HdiP7OEss96g6SMf6T_o_gwCJHO3y_HMt1Vsd0Dcd9g err="memcache: connect timeout to 10.11.105.37:11211" | |
| level=warn ts=2021-01-31T23:52:49.124703823Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX5TFQ4GD5M9W1GX5YWTBBJ7:2AfHX7ruDuOcnvb9fQ1D-Ay6KS7_gCblT8qLuXP4LB4 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763770747Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FVGEA3V1DM2AB3DNM4G6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763839865Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8FVGEA3V1DM2AB3DNM4G6:suiPY_V3l3jnZeOc38n7buBjA6z0U5i9dRsQfWFBpkw err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763862013Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1MGY3RF5K2XY99PK90D82:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763863399Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8G6KYK22JH23N91P5VHXY:xkiTXn85xh1W-b_3EavQ9Spk_iWhzn5vEa8p4Xay9Z8 err="read tcp 10.11.247.204:34794->10.11.224.48:11211: i/o timeout" | |
| level=warn ts=2021-01-31T23:52:53.763874357Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRCK8B3NKE4J02XG66MNN:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763891909Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FJD6YEGTPQAQG8PX5EAV:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763907949Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1KXVMPK65MFG2TJWFZ5AV:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763923659Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRC7MPXF682ATY9JE31QR:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763940839Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8G6KYK22JH23N91P5VHXY:suiPY_V3l3jnZeOc38n7buBjA6z0U5i9dRsQfWFBpkw err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763957665Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRAN50VS5281ENAZDDGN8:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763961991Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRC7MPXF682ATY9JE31QR:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763965213Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FVGEA3V1DM2AB3DNM4G6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763954319Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRAN50VS5281ENAZDDGN8:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763971994Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8FJD6YEGTPQAQG8PX5EAV:rRsC3NwUNO4X4dFCQ0zE-ylNsBjT_aZAst3TkW_SGRk err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763997463Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G60TNT35BDACM9N250Y6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764005472Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EWVGX1N0RDW6937SFFN95PAW:LwNKvQteHuEsb9i-lyH5jCSeicCE04zRbCQKWAp1xpw err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764019409Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G60TNT35BDACM9N250Y6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764039138Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRC7MPXF682ATY9JE31QR:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764039309Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8G6KYK22JH23N91P5VHXY:rRsC3NwUNO4X4dFCQ0zE-ylNsBjT_aZAst3TkW_SGRk err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764049199Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTQVT3S1C1QD3DS6DNF69Q:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764068849Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G6KYK22JH23N91P5VHXY:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764152295Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTQVT3S1C1QD3DS6DNF69Q:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.76414745Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRCK8B3NKE4J02XG66MNN:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763785389Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FBSGADA327QZZG83QSVJ:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763867406Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8G60TNT35BDACM9N250Y6:suiPY_V3l3jnZeOc38n7buBjA6z0U5i9dRsQfWFBpkw err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764269744Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G60TNT35BDACM9N250Y6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764280874Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=4 firstKey=S:01EWVGWNGF9M6YM3HNFWPD8F0K:272405792 err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764276898Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1K7XEJTXCTS2TGNK0G5GN:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763873076Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1MGY3RF5K2XY99PK90D82:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.763786581Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=1 firstKey=P:01EXC8F0C7J4N7CQNXFEA0QKC3:suiPY_V3l3jnZeOc38n7buBjA6z0U5i9dRsQfWFBpkw err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764865543Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTQVT3S1C1QD3DS6DNF69Q:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764896906Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FJD6YEGTPQAQG8PX5EAV:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764947724Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FVGEA3V1DM2AB3DNM4G6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.765075596Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1K2CGEDF5V8P350G2ECDQ:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764093088Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTRAN50VS5281ENAZDDGN8:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764894795Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FBSGADA327QZZG83QSVJ:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.76490403Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXBTR3X6VY51RNHVXTK5QHQ3:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764907674Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8FVGEA3V1DM2AB3DNM4G6:d1ZeqQ4Fdn5fPh_efkAK0cQ4waO0iVxW6s0HZc_6_Kg err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.764903319Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G60TNT35BDACM9N250Y6:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.765914622Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G6KYK22JH23N91P5VHXY:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.766098672Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1MGY3RF5K2XY99PK90D82:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.766182315Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC8G6KYK22JH23N91P5VHXY:d1ZeqQ4Fdn5fPh_efkAK0cQ4waO0iVxW6s0HZc_6_Kg err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=warn ts=2021-01-31T23:52:53.766479866Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EXC1K2CGEDF5V8P350G2ECDQ:nnEIuoDigcX3vP1zw_GKoQc8_CqkkB-n8rzgN4Q_rwc err="memcache: connect timeout to 10.11.224.48:11211" | |
| level=info ts=2021-01-31T23:52:53.766872978Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.682736805s id=01EVPKVVKRH75M36D6VW5CVMPM | |
| level=info ts=2021-01-31T23:52:53.769380728Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.661331905s id=01ESAE9SPZJ4GXE822XR7RDHP1 | |
| level=info ts=2021-01-31T23:52:53.771182958Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.658112532s id=01ETJJ906YZEHCT5JTZKTQ5BHD | |
| level=info ts=2021-01-31T23:52:53.772478432Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.090096795s id=01EV2N9ACCXKGTMK73ET72YPH6 | |
| level=info ts=2021-01-31T23:52:53.773633809Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.651508868s id=01EWP57PCCCJABH6R640B0CFYX | |
| level=info ts=2021-01-31T23:52:53.774850455Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.653854616s id=01EWC2H10CY2KWJZ8R4721G5NE | |
| level=info ts=2021-01-31T23:52:53.776843081Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.096716042s id=01ESKENM5VZMZKATH83AA1FRFE | |
| level=info ts=2021-01-31T23:52:53.777953262Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.099035187s id=01EWRQMAB7RH2W2RKQ6W1073Z4 | |
| level=info ts=2021-01-31T23:52:53.779012755Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.065480172s id=01ESS15T23VACNP4MN5H7SBG75 | |
| level=info ts=2021-01-31T23:52:53.780078502Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.098807721s id=01EWQN8YJHK6VB7VSTCKV1FBWN | |
| level=info ts=2021-01-31T23:52:53.781145097Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.695692694s id=01ETX9MSM6CZ73Y18QHRVRV2ET | |
| level=info ts=2021-01-31T23:52:53.782561274Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.099107346s id=01ETHPVN1DDJK24WJ3QFGZR3RC | |
| level=info ts=2021-01-31T23:52:53.783693Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.671786237s id=01EX2KFVMY8KGER2JREW2PVW1Z | |
| level=info ts=2021-01-31T23:52:53.784690436Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.670507963s id=01ETDDH7NKRAZX26R2GH2H3T2J | |
| level=info ts=2021-01-31T23:52:53.785688648Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.673801311s id=01ESTH8GG3VQYNNRC7043EVFSG | |
| level=info ts=2021-01-31T23:52:53.788919209Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.68316134s id=01ET8FJRAD3CBW1MGB2V8XHRQQ | |
| level=info ts=2021-01-31T23:52:53.790890495Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.076223951s id=01EVWD8XATGH9VK04NPCPPWBKR | |
| level=info ts=2021-01-31T23:52:53.792424429Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.689721993s id=01EWDJJE8GJ9NPGSV2701EGFQK | |
| level=info ts=2021-01-31T23:52:53.793515013Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.686589394s id=01ES7E4SCMM95A13TG278JRWW3 | |
| level=info ts=2021-01-31T23:52:53.794719368Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=12.110171426s id=01EV5NE0RMHH5WA83SP4DEDDT3 | |
| level=warn ts=2021-01-31T23:52:53.984233208Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=2 firstKey=P:01EX6WTBEN6NA9FX56H0WQ83HZ:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:53.990454937Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=8917 firstKey=S:01EX6WTBEN6NA9FX56H0WQ83HZ:576939840 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:53.995593297Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=8917 firstKey=S:01EX6WTBEN6NA9FX56H0WQ83HZ:576939840 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:53.995607641Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=8917 firstKey=S:01EX6WTBEN6NA9FX56H0WQ83HZ:576939840 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.010659601Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX6NXS9NWZ3BX9029Z3N2SVG:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.010910784Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX6NXS9NWZ3BX9029Z3N2SVG:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.021029363Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9577 firstKey=S:01EX6NXS9NWZ3BX9029Z3N2SVG:625475024 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.021047667Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9577 firstKey=S:01EX6NXS9NWZ3BX9029Z3N2SVG:625475024 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.021056083Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9577 firstKey=S:01EX6NXS9NWZ3BX9029Z3N2SVG:625475024 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.029911283Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX6NY4A7AR7ERWQQ5APZM1CW:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.032765809Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX6NY4A7AR7ERWQQ5APZM1CW:vsUF6BJTuUhH-wrMrmMH3eYytZqfH20jYLHTj9cDaMw err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.040005998Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=10074 firstKey=S:01EX6NY4A7AR7ERWQQ5APZM1CW:641807264 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.040023991Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=10074 firstKey=S:01EX6NY4A7AR7ERWQQ5APZM1CW:641807264 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:52:54.040022716Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=10074 firstKey=S:01EX6NY4A7AR7ERWQQ5APZM1CW:641807264 err="failed to wait for turn: context canceled" | |
| level=info ts=2021-01-31T23:52:59.304092367Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=5.537193411s id=01ETC4AMS016PYYPH8VEQ1C40H | |
| level=info ts=2021-01-31T23:52:59.980791465Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.195083721s id=01EWQ7HRF2B22PSYRNA2R4ZGDE | |
| level=warn ts=2021-01-31T23:53:00.271536761Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=30.78336509s err="rpc error: code = Aborted desc = fetch series for block 01EX6WTBEN6NA9FX56H0WQ83HZ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:00.271959402Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=30.776249964s err="rpc error: code = Aborted desc = fetch series for block 01EX6WTBEN6NA9FX56H0WQ83HZ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:00.278149175Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9172 firstKey=S:01EX6WT228JD2B47QCSAD8EVWJ:588818672 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:00.278987283Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9172 firstKey=S:01EX6WT228JD2B47QCSAD8EVWJ:588818672 err="failed to wait for turn: context canceled" | |
| level=info ts=2021-01-31T23:53:01.300409956Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.529201188s id=01EW628KJT16DX1S2FHD2BMXHC | |
| level=info ts=2021-01-31T23:53:01.301566267Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.526688257s id=01EVM1EXJG4ADWJBM5WHCFFP1J | |
| level=info ts=2021-01-31T23:53:01.302666842Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.53325496s id=01EW997E46P323CCMBMVT843TA | |
| level=info ts=2021-01-31T23:53:01.303784171Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.512862358s id=01EW4ZXP0D6GEH87XZ52DWND9B | |
| level=info ts=2021-01-31T23:53:01.332088157Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.546382986s id=01ETSVS9DMCQK681J1Z6V1XDNK | |
| level=info ts=2021-01-31T23:53:01.333263892Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.560761318s id=01ETXGF51Z92VF66EH893HQW5Z | |
| level=info ts=2021-01-31T23:53:01.334319239Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.555280461s id=01ES17180MWHNYK41TBWVNSNKH | |
| level=info ts=2021-01-31T23:53:01.360727251Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.567191924s id=01EX73PWAYQQVEVZ8YC53ZTZ9S | |
| level=info ts=2021-01-31T23:53:01.361855435Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.583869536s id=01EX9WYTHKDWZGKMT2WK4KFF7Q | |
| level=info ts=2021-01-31T23:53:01.362963779Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.574003151s id=01ERZG3Q418ZRC1BXRFF86Z26P | |
| level=info ts=2021-01-31T23:53:01.364041152Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.58393647s id=01ES9BYG0HGAWFJ4FQ8A3PXRPM | |
| level=info ts=2021-01-31T23:53:01.365117605Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.582549701s id=01EV87T6HHS45ASFSKGVYAQ3GQ | |
| level=info ts=2021-01-31T23:53:01.366215253Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.589333134s id=01EW998RT6K6CA8AKAA9EKGDBF | |
| level=info ts=2021-01-31T23:53:01.367438858Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.593779957s id=01ET81TVX8FGXACFDGDNZP624X | |
| level=info ts=2021-01-31T23:53:01.368484592Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.574949468s id=01ETN4PTE86W8TC707Z9BWA7WM | |
| level=info ts=2021-01-31T23:53:01.369544037Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.58383828s id=01EW1B6519PS5X9PE9CJZBMKHQ | |
| level=info ts=2021-01-31T23:53:01.370739811Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.575983741s id=01ESMYRC41KN2TSQNQEPVFRTDN | |
| level=info ts=2021-01-31T23:53:01.371902483Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.589311592s id=01ES7VX69HAKM8CV20RJWP2ZGK | |
| level=warn ts=2021-01-31T23:53:02.169493521Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=32.652926732s err="rpc error: code = Aborted desc = fetch series for block 01EX6WTBEN6NA9FX56H0WQ83HZ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169489567Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.142189685s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169485735Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.745802508s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY2XGE86FHMXH3MYTGWK8: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169603563Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.795859248s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169614131Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.794820916s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY2XGE86FHMXH3MYTGWK8: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.16964577Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.795667744s err="rpc error: code = Aborted desc = fetch series for block 01EX6WV3N90D6EHDNWABMHGAH3: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169662031Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.795736079s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY2XGE86FHMXH3MYTGWK8: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169684868Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.716757542s err="rpc error: code = Aborted desc = fetch series for block 01EX6NZDPY72D401FGZAYVPV2Q: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169703411Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=3 firstKey=P:01EX6NY54S22H6G4ME9VMTS3XP:0U6Adltjl3JF0DL5BdJjKuGoFR-3enALMYjXypyynE8 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:02.169731457Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.794971412s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169694574Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=32.678428732s err="rpc error: code = Aborted desc = fetch series for block 01EX6WTBEN6NA9FX56H0WQ83HZ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169782196Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.796713545s err="rpc error: code = Aborted desc = fetch series for block 01EX6NZDPY72D401FGZAYVPV2Q: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169806696Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.795883162s err="rpc error: code = Aborted desc = fetch series for block 01EX6NYYZ4VGXFF2WBJD2AWE2Q: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169811986Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.719818415s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169841926Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.796118612s err="rpc error: code = Aborted desc = fetch series for block 01EX6NXT5NF3TZ8MA2RFWAMZK2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169872208Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.795703812s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSVJ7AP4123R29950SRAF: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169879023Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.720072434s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY2XGE86FHMXH3MYTGWK8: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169889678Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.706845442s err="rpc error: code = Aborted desc = fetch series for block 01EX6WT9D70VSW7XC00Q6Z3QFH: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169890048Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=32.67862881s err="rpc error: code = Aborted desc = fetch series for block 01EX6WTBEN6NA9FX56H0WQ83HZ: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.169923849Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.6906001s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.170005903Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.711585809s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY2XGE86FHMXH3MYTGWK8: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.170011038Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.716455173s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.170026347Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.716644191s err="rpc error: code = Aborted desc = fetch series for block 01EX6NZFFWWJC4ZQ4QJVCSBJ4V: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.170036975Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.720270912s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.18328307Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9611 firstKey=S:01EX6NY54S22H6G4ME9VMTS3XP:625084176 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:02.183382583Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9611 firstKey=S:01EX6NY54S22H6G4ME9VMTS3XP:625084176 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:02.183408569Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=9611 firstKey=S:01EX6NY54S22H6G4ME9VMTS3XP:625084176 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:02.185124627Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.811075498s err="rpc error: code = Aborted desc = fetch series for block 01EX6NXT5NF3TZ8MA2RFWAMZK2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.185507226Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=32.66900556s err="rpc error: code = Aborted desc = fetch series for block 01EX6WTBEN6NA9FX56H0WQ83HZ: preload series: read series range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:02.18563045Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=23.736184887s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=info ts=2021-01-31T23:53:07.907773503Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.544783935s id=01EVDTASMZ0XH6S60HBYC6Q99N | |
| level=info ts=2021-01-31T23:53:07.909384086Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.548626094s id=01EW8MN2DVV24250XS3511GWQM | |
| level=info ts=2021-01-31T23:53:07.910409675Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.57605358s id=01ETDMBVMFS6TS768R1J6BNDHR | |
| level=info ts=2021-01-31T23:53:07.911490885Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.539558177s id=01ETNJD6QFN4GMB9S37Q235N0S | |
| level=info ts=2021-01-31T23:53:07.916548494Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=8.612384613s id=01ETADCYA9DZ8S5C4EZRV6HSQ0 | |
| level=info ts=2021-01-31T23:53:07.917701974Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.585576961s id=01EW44FCNPEED0NQQ5TPF8BP5B | |
| level=info ts=2021-01-31T23:53:07.918727909Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.585416912s id=01ETT2KPED99K1WERM0S15AP8N | |
| level=info ts=2021-01-31T23:53:07.919762003Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.55787885s id=01EWZT6TPDBZ7XMQ56661M77QQ | |
| level=info ts=2021-01-31T23:53:07.920900907Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.617083905s id=01ESRKEXCC7XTFYMHZM1EQBN5J | |
| level=info ts=2021-01-31T23:53:07.921938288Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.621491023s id=01ES29B3645DAH1KV62MVDME3G | |
| level=info ts=2021-01-31T23:53:07.922980981Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.557838555s id=01EWZKCHZ1W7D6TCV24HZX0CXV | |
| level=info ts=2021-01-31T23:53:07.924022457Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.556561141s id=01EVACG6J9RH2PYAFV3KX37JPM | |
| level=info ts=2021-01-31T23:53:07.925065127Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.623470432s id=01ETK6XNT7W52EES47YHS7R444 | |
| level=info ts=2021-01-31T23:53:07.926111578Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.557603337s id=01ESN5JYC6H96S2QKY8Q41JXG9 | |
| level=info ts=2021-01-31T23:53:07.927137524Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.557569751s id=01ETEFVS5KCN47KG5RRNP3HDFE | |
| level=info ts=2021-01-31T23:53:07.928235858Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.564171931s id=01ET7D819HB0C9N9DDVTH33BP9 | |
| level=info ts=2021-01-31T23:53:07.929227408Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.626533007s id=01ETXQAVSG09CXPVE7H6RWQZPS | |
| level=info ts=2021-01-31T23:53:07.930356225Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=7.949525795s id=01ET3ARKEMR9FP6AFVPK63H72T | |
| level=info ts=2021-01-31T23:53:07.931454252Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.565216595s id=01EW01ZX7688SV2VYPMH0445W7 | |
| level=info ts=2021-01-31T23:53:07.932562182Z caller=bucket.go:484 org_id=fake msg="loaded new block" elapsed=6.561798366s id=01EW323Y9M6JE7AG43ZWAYATWD | |
| level=warn ts=2021-01-31T23:53:08.021277489Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=22.608711756s err="rpc error: code = Unknown desc = send series response hints: rpc error: code = Internal desc = transport: transport: the stream is done or WriteHeader was already called" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:08.203665415Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=6 firstKey=P:01EX316FK7C1RK83GWT3BTPYYX:2gmaL7FyTFFkWrLxlPTZxJwOoDLrkkYyETqcp0l1FL4 err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:08.251629205Z caller=memcached_client.go:382 msg="failed to fetch items from memcached" numKeys=5 firstKey=P:01EX25RERPA9JCC1JVWNFSDMHB:HdiP7OEss96g6SMf6T_o_gwCJHO3y_HMt1Vsd0Dcd9g err="failed to wait for turn: context canceled" | |
| level=warn ts=2021-01-31T23:53:08.375703372Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=29.214554403s err="rpc error: code = Aborted desc = fetch series for block 01EX6NYYZV9BGPKS0GJ3JSJZC2: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:08.377247855Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=29.102000084s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY6TP74K6RGH9RKH1FVEM: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:08.446715043Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=30.072886122s err="rpc error: code = Aborted desc = fetch series for block 01EX6WSYJWHRKSQKBFC9QAVVZT: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:08.447087623Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=30.073224601s err="rpc error: code = Aborted desc = fetch series for block 01EX6NY2XGE86FHMXH3MYTGWK8: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| level=warn ts=2021-01-31T23:53:08.448774636Z caller=grpc_logging.go:55 method=/gatewaypb.StoreGateway/Series duration=30.025163977s err="rpc error: code = Aborted desc = fetch series for block 01EX6WV1QPZB49F281T8WN0QVA: expanded matching posting: get postings: read postings range: get range reader: context canceled" msg="gRPC\n" | |
| unexpected fault address 0x7f3e16fe7905 | |
| fatal error: fault | |
| [signal SIGSEGV: segmentation violation code=0x1 addr=0x7f3e16fe7905 pc=0x4020ac] | |
| goroutine 10902586 [running]: | |
| runtime.throw(0x2533220, 0x5) | |
| /usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc0e3094f38 sp=0xc0e3094f08 pc=0x434de2 | |
| runtime.sigpanic() | |
| /usr/local/go/src/runtime/signal_unix.go:702 +0x3cc fp=0xc0e3094f68 sp=0xc0e3094f38 pc=0x44bb8c | |
| cmpbody() | |
| /usr/local/go/src/internal/bytealg/compare_amd64.s:115 +0xec fp=0xc0e3094f70 sp=0xc0e3094f68 pc=0x4020ac | |
| github.com/thanos-io/thanos/pkg/block/indexheader.BinaryReader.postingsOffset(0x2b73f00, 0xc1182fc5a0, 0xc114ce36d0, 0x2b44ce0, 0xc1182fc560, 0xc0b4452180, 0x0, 0xc0d7977140, 0xc10c1ba420, 0xc353b7c190, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:696 +0xe54 fp=0xc0e30950e8 sp=0xc0e3094f70 pc=0x18f5b94 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.BinaryReader.PostingsOffset(0x2b73f00, 0xc1182fc5a0, 0xc114ce36d0, 0x2b44ce0, 0xc1182fc560, 0xc0b4452180, 0x0, 0xc0d7977140, 0xc10c1ba420, 0xc353b7c190, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:646 +0xbf fp=0xc0e30951c8 sp=0xc0e30950e8 pc=0x18f4b6f | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc020951520, 0x15, 0x7f3e16fe7905, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:162 +0x1b3 fp=0xc0e3095318 sp=0xc0e30951c8 pc=0x18f7db3 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00b6e1c80, 0xc093896e00, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 fp=0xc0e3095578 sp=0xc0e3095318 pc=0x199db71 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00b6e1c80, 0xc00b9642e0, 0x2, 0x2, 0xc182bf6120, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac fp=0xc0e3095818 sp=0xc0e3095578 pc=0x199b68c | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc00b6e1c80, 0xc11d7a3f80, 0xc00b9642e0, 0x2, 0x2, 0xc11d7a2540, 0x2b44f80, 0xc02b22f530, 0x125fde00, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a fp=0xc0e3095d00 sp=0xc0e3095818 pc=0x19931ba | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x125ff840, 0x125ff8a0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe fp=0xc0e3095f78 sp=0xc0e3095d00 pc=0x19a373e | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc02b22f4d0, 0xc0829de0e0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 fp=0xc0e3095fd0 sp=0xc0e3095f78 pc=0x997bc9 | |
| runtime.goexit() | |
| /usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc0e3095fd8 sp=0xc0e3095fd0 pc=0x467ca1 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 1 [select, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*Manager).AwaitStopped(0xc000dfb2c0, 0x2b99cc0, 0xc00005c488, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/util/services/manager.go:145 +0xbb | |
| github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run(0xc001003500, 0xc0002fe200, 0x4) | |
| /__w/cortex/cortex/pkg/cortex/cortex.go:416 +0x91c | |
| main.main() | |
| /__w/cortex/cortex/cmd/cortex/main.go:189 +0xe3b | |
| goroutine 158 [select]: | |
| go.opencensus.io/stats/view.(*worker).start(0xc000492800) | |
| /__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:276 +0x100 | |
| created by go.opencensus.io/stats/view.init.0 | |
| /__w/cortex/cortex/vendor/go.opencensus.io/stats/view/worker.go:34 +0x68 | |
| goroutine 177 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/alertmanager.init.0.func1() | |
| /__w/cortex/cortex/pkg/alertmanager/alertmanager.go:95 +0x55 | |
| created by github.com/cortexproject/cortex/pkg/alertmanager.init.0 | |
| /__w/cortex/cortex/pkg/alertmanager/alertmanager.go:91 +0x35 | |
| goroutine 283 [chan receive, 62 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc000dfb500, 0x2b7fac0, 0xc000817760) | |
| /__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener | |
| /__w/cortex/cortex/pkg/util/services/manager.go:243 +0xfb | |
| goroutine 261 [chan receive]: | |
| go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc000a07a20) | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5 | |
| created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80 | |
| goroutine 303 [chan receive]: | |
| go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0004859a0) | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5 | |
| created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80 | |
| goroutine 306 [chan receive]: | |
| go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc000485bc0) | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5 | |
| created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger | |
| /__w/cortex/cortex/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80 | |
| goroutine 282 [chan receive, 62 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000dfb440, 0x2ba22c0, 0xc000817680) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 344 [select]: | |
| github.com/uber/jaeger-client-go/utils.(*reconnectingUDPConn).reconnectLoop(0xc000232700, 0x6fc23ac00) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:70 +0xed | |
| created by github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x10f | |
| goroutine 376 [select]: | |
| github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollControllerWithTicker(0xc000c95d40, 0xc00017ac30) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:144 +0xd0 | |
| github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollController(0xc000c95d40) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:139 +0x6e | |
| created by github.com/uber/jaeger-client-go.NewRemotelyControlledSampler | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:86 +0x119 | |
| goroutine 281 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000dfb3e0, 0x2ba22c0, 0xc000817640) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 345 [select]: | |
| github.com/uber/jaeger-client-go.(*remoteReporter).processQueue(0xc000179560) | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:296 +0x120 | |
| created by github.com/uber/jaeger-client-go.NewRemoteReporter | |
| /__w/cortex/cortex/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x1a0 | |
| goroutine 488 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549 | |
| goroutine 393 [syscall, 63 minutes]: | |
| os/signal.signal_recv(0x26b85d0) | |
| /usr/local/go/src/runtime/sigqueue.go:147 +0x9c | |
| os/signal.loop() | |
| /usr/local/go/src/os/signal/signal_unix.go:23 +0x22 | |
| created by os/signal.Notify.func1 | |
| /usr/local/go/src/os/signal/signal.go:127 +0x44 | |
| goroutine 489 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 490 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 491 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 492 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 493 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 494 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 495 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 496 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 497 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 498 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 499 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 500 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 501 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 502 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 503 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 504 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 505 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 506 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 507 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 508 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 509 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 510 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 511 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 512 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 513 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 514 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 515 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 516 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 517 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 518 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 519 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 520 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec3de3870, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc17b004418, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc17b004400, 0xc0caccf000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc17b004400, 0xc0caccf000, 0x1000, 0x1000, 0x3f3fee0, 0xc122107b40, 0x6fe69e) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02c9e66e8, 0xc0caccf000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).fill(0xc14a7f6c00) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).ReadSlice(0xc14a7f6c00, 0xa, 0x0, 0x2, 0x2, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:359 +0x3d | |
| github.com/bradfitz/gomemcache/memcache.(*Client).populateOne(0xc0002ff140, 0xc0cacb2980, 0x25310e6, 0x3, 0xc0a9221140, 0x18cdf80, 0xc0fee18c60) | |
| /__w/cortex/cortex/vendor/github.com/bradfitz/gomemcache/memcache/memcache.go:587 +0x312 | |
| github.com/bradfitz/gomemcache/memcache.(*Client).set(0xc0002ff140, 0xc0cacb2980, 0xc0a9221140, 0xc0c6b5b170, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/bradfitz/gomemcache/memcache/memcache.go:525 +0x54 | |
| github.com/bradfitz/gomemcache/memcache.(*Client).onItem(0xc0002ff140, 0xc0a9221140, 0x26b3018, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/bradfitz/gomemcache/memcache/memcache.go:308 +0x11b | |
| github.com/bradfitz/gomemcache/memcache.(*Client).Set(0xc0002ff140, 0xc0a9221140, 0x3f6bae0, 0x4e94914f0000) | |
| /__w/cortex/cortex/vendor/github.com/bradfitz/gomemcache/memcache/memcache.go:521 +0x41 | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).SetAsync.func1() | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:343 +0x1c9 | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:527 +0x78 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 521 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 522 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 523 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 524 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 525 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 526 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 527 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 528 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 529 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 530 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 531 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 532 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 533 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 534 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 535 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 536 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 537 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 538 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7770) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 615 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549 | |
| goroutine 617 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 616 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 618 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 619 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 620 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 621 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 622 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 623 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 624 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 625 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 626 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 627 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 628 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 629 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 630 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 631 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 632 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 633 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 634 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 635 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 636 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 637 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 638 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 639 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 640 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 641 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 642 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 643 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 644 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 645 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 646 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 647 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 648 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 649 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 650 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 651 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 652 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 653 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 654 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 655 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 656 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 657 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 658 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 659 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 660 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 661 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 662 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 663 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 664 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 665 [select, 5 minutes]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc0003b7860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 747 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).resolveAddrsLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:541 +0x136 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:313 +0x1549 | |
| goroutine 749 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 748 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 750 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 751 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 752 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 753 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 754 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 755 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 756 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 757 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 758 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 759 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 760 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 761 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 762 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 763 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 764 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 765 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 766 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 767 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 768 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 769 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 770 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 771 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 772 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 773 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 774 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 775 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 776 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 777 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 778 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 779 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 780 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 781 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 782 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 783 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 784 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 785 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 786 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 787 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 788 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 789 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 790 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 791 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 792 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 793 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 794 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 795 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 796 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 797 [select]: | |
| github.com/thanos-io/thanos/pkg/cacheutil.(*memcachedClient).asyncQueueProcessLoop(0xc001138000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:525 +0x102 | |
| created by github.com/thanos-io/thanos/pkg/cacheutil.newMemcachedClient | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/cacheutil/memcached_client.go:319 +0x1594 | |
| goroutine 279 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000dfb320, 0x2ba22c0, 0xc000817580) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 280 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000dfb380, 0x2ba22c0, 0xc0008175e0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 284 [select, 63 minutes]: | |
| github.com/weaveworks/common/signals.(*Handler).Loop(0xc000dfd110) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/signals/signals.go:48 +0x1d7 | |
| github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run.func4(0xc000dfd110, 0xc000dfb2c0) | |
| /__w/cortex/cortex/pkg/cortex/cortex.go:405 +0x2b | |
| created by github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run | |
| /__w/cortex/cortex/pkg/cortex/cortex.go:404 +0x71b | |
| goroutine 285 [select, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000734d80, 0x2b99c80, 0xc000216b80, 0x4, 0xc00017c5a0, 0xc0011cc601, 0xc00065ee88) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000734d80, 0x2b99c80, 0xc000216b80, 0xc000dfb320, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc0002fe2c0, 0x2b99c80, 0xc000216b80, 0x26b85c8, 0xc000734e28) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000734e10) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 286 [select, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000734ea0, 0x2b99c80, 0xc000216bc0, 0x4, 0xc00017c840, 0xc000dfb380, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000734ea0, 0x2b99c80, 0xc000216bc0, 0x0, 0xc000734f30) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc0002fe440, 0x2b99c80, 0xc000216bc0, 0x26b85c8, 0xc000734f48) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000734f30) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 287 [select, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000735050, 0x2b99c80, 0xc000216c00, 0x4, 0xc00017c9c0, 0xc0011cc601, 0xc001031e88) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000735050, 0x2b99c80, 0xc000216c00, 0xc000dfb3e0, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc0002fe4c0, 0x2b99c80, 0xc000216c00, 0x26b85c8, 0xc0007350f8) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0007350e0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 288 [select, 62 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc00072c090, 0x2b99c80, 0xc000216c80, 0x4, 0xc000dfe120, 0x1, 0xc001030e88) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:271 +0xcc | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc00072c090, 0x2b99c80, 0xc000216c80, 0xc000dfb440, 0x26b3980) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:267 +0x51 | |
| github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000216ac0, 0x2b99c80, 0xc000216c80, 0x26b85c8, 0xc00072c138) | |
| /__w/cortex/cortex/pkg/util/module_service.go:70 +0x48 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc00072c120) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 817 [select, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1(0x2b99c80, 0xc000216e40, 0x2, 0xc0011bc768) | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:28 +0xfb | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000734ea0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 818 [chan receive, 63 minutes]: | |
| github.com/weaveworks/common/server.(*Server).Run(0xc00004d340, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:414 +0x15b | |
| github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1.1(0xc000179aa0, 0xc00004d340) | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:25 +0x57 | |
| created by github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1 | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:23 +0x5e | |
| goroutine 819 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/cortex.ignoreSignalHandler.Loop(0xc00017c6c0) | |
| /__w/cortex/cortex/pkg/cortex/server_service.go:65 +0x34 | |
| github.com/weaveworks/common/server.(*Server).Run.func1(0xc00004d340, 0xc000dfb800) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:374 +0x3a | |
| created by github.com/weaveworks/common/server.(*Server).Run | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:373 +0x6b | |
| goroutine 820 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5784190d58, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000493218, 0x72, 0x0, 0x0, 0x253657d) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc000493200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4 | |
| net.(*netFD).accept(0xc000493200, 0x4c9469fd1d8cb872, 0x1000000000000, 0x4c9469fd1d8cb872) | |
| /usr/local/go/src/net/fd_unix.go:238 +0x42 | |
| net.(*TCPListener).accept(0xc00036ff20, 0x601742e2, 0xc000cfce08, 0x4db7f6) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00036ff20, 0xc000cfce58, 0x18, 0xc0011ccc00, 0x8156cc) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x64 | |
| net/http.(*Server).Serve(0xc00041e2a0, 0x2b87e80, 0xc00036ff20, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2930 +0x25d | |
| github.com/weaveworks/common/server.(*Server).Run.func2(0xc00004d340, 0xc000dfb800) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:384 +0x126 | |
| created by github.com/weaveworks/common/server.(*Server).Run | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:381 +0x97 | |
| goroutine 821 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5784190c78, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000493418, 0x72, 0x0, 0x0, 0x253657d) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc000493400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4 | |
| net.(*netFD).accept(0xc000493400, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:238 +0x42 | |
| net.(*TCPListener).accept(0xc00036ff40, 0xc00065be50, 0xc00065be58, 0x18) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00036ff40, 0x26b6670, 0xc000c0cea0, 0x2bc4400, 0xc165d3ea68) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x64 | |
| google.golang.org/grpc.(*Server).Serve(0xc000c0cea0, 0x2b87e80, 0xc00036ff40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:621 +0x210 | |
| github.com/weaveworks/common/server.(*Server).Run.func3(0xc00004d340, 0xc000dfb800) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:403 +0x4e | |
| created by github.com/weaveworks/common/server.(*Server).Run | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/server/server.go:402 +0x13b | |
| goroutine 804 [select]: | |
| github.com/cortexproject/cortex/pkg/util/runtimeconfig.(*Manager).loop(0xc000734fc0, 0x2b99c80, 0xc0005d4a00, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/util/runtimeconfig/manager.go:132 +0x13f | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000735050) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 833 [select, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).running(0xc0001796e0, 0x2b99c80, 0xc000664080, 0x26b85c8, 0xc000734d98) | |
| /__w/cortex/cortex/pkg/ring/kv/memberlist/kv_init_service.go:52 +0xd6 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000734d80) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 849 [semacquire, 5 minutes]: | |
| sync.runtime_Semacquire(0xc1b0644198) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1b0644190) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks(0xc0001cf600, 0x2b99c80, 0xc000fa0080, 0x26b38d8, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:225 +0x53e | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).SyncBlocks(0xc0001cf600, 0x2b99c80, 0xc000fa0080, 0x1, 0x2bb6b40) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:138 +0x4b | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).syncStores(0xc00083ae00, 0x2b99c80, 0xc000fa0080, 0x253ea92, 0xb) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:312 +0x23e | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).running(0xc00083ae00, 0x2b99c80, 0xc000fa0080, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:291 +0x31e | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc00072c090) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 850 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000f941e0, 0x2ba22c0, 0xc000fa2060) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 851 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000f94240, 0x2ba22c0, 0xc000fa20a0) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:321 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:320 +0x11c | |
| goroutine 852 [chan receive, 63 minutes]: | |
| github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc000f942a0, 0x2b7fac0, 0xc000fa20e0) | |
| /__w/cortex/cortex/pkg/util/services/manager.go:244 +0x66 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener | |
| /__w/cortex/cortex/pkg/util/services/manager.go:243 +0xfb | |
| goroutine 853 [select]: | |
| github.com/cortexproject/cortex/pkg/ring.(*BasicLifecycler).running(0xc000737110, 0x2b99c80, 0xc000fa00c0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/ring/basic_lifecycler.go:189 +0x18a | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000735680) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 854 [select]: | |
| golang.org/x/time/rate.(*Limiter).WaitN(0xc001484050, 0x2b99d40, 0xc000f98120, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/time/rate/rate.go:259 +0x31a | |
| golang.org/x/time/rate.(*Limiter).Wait(...) | |
| /__w/cortex/cortex/vendor/golang.org/x/time/rate/rate.go:219 | |
| github.com/cortexproject/cortex/pkg/ring/kv/consul.(*Client).WatchKey(0xc00020d030, 0x2b99d40, 0xc000f98120, 0x2542ee3, 0xd, 0xc000f920d0) | |
| /__w/cortex/cortex/pkg/ring/kv/consul/client.go:210 +0x1a2 | |
| github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey.func1(0x2b99d40, 0xc000f98120, 0x8, 0xbffdeae78acbd85c) | |
| /__w/cortex/cortex/pkg/ring/kv/metrics.go:91 +0x5e | |
| github.com/weaveworks/common/instrument.CollectedRequest(0x2b99c80, 0xc000fa0100, 0x25384df, 0x8, 0x2b846c0, 0xc0001a7a28, 0x26b5d20, 0xc0005abdf0, 0x0, 0x10) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:152 +0x269 | |
| github.com/cortexproject/cortex/pkg/ring/kv.metrics.WatchKey(0x2bb65a0, 0xc00020d030, 0xc0001a7a28, 0x2b99c80, 0xc000fa0100, 0x2542ee3, 0xd, 0xc000f920d0) | |
| /__w/cortex/cortex/pkg/ring/kv/metrics.go:90 +0x101 | |
| github.com/cortexproject/cortex/pkg/ring.(*Ring).loop(0xc000388b40, 0x2b99c80, 0xc000fa0100, 0x26b85c8, 0xc000735728) | |
| /__w/cortex/cortex/pkg/ring/ring.go:235 +0x95 | |
| github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000735710) | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:167 +0x3f7 | |
| created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1 | |
| /__w/cortex/cortex/pkg/util/services/basic_service.go:96 +0xa8 | |
| goroutine 920 [select]: | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewReaderPool.func1(0xc000fa0b00, 0x1bf08eb000) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:54 +0xd2 | |
| created by github.com/thanos-io/thanos/pkg/block/indexheader.NewReaderPool | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:52 +0x13e | |
| goroutine 10898138 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc165902f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10850787 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783181f40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0435ef018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0435ef000, 0xc0c7a4c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0435ef000, 0xc0c7a4c000, 0x8000, 0x8000, 0x203029, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc5316d00f8, 0xc0c7a4c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc14b6878c0, 0xc068c4db58, 0x9, 0x9, 0xc065237880, 0x1007f57aae825d8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc14b6878c0, 0xc068c4db58, 0x9, 0x9, 0x9, 0x601742dd, 0xc124706dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc068c4db58, 0x9, 0x9, 0x2b41140, 0xc14b6878c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc068c4db20, 0xc18a438e80, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0a5e3c780, 0xc144b08720, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10899876 [semacquire]: | |
| sync.runtime_Semacquire(0xc0b50e71b0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0b50e71a8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0b50e71a0, 0xc09496ce10, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc226869ac0, 0xc0fb86c500, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1882 +0x863 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc226869ac0, 0xc00c6be840, 0x2, 0x2, 0xc176c82d20, 0xd, 0x1, 0xc002c80a80, 0x7f57aae887c0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0076a5680, 0xc226869ac0, 0xc0dd83b920, 0xc00c6be840, 0x2, 0x2, 0xc0dd83a2a0, 0x2b44f80, 0xc149156f90, 0x721a6369706f7405, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x9fa8746e6576655f, 0x706f74050201c884) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc149156f60, 0xc02af70e70) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942644 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc165aead80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc02b217f20, 0xc1ff1a0420, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc02b217f20, 0xc1ff1a0420, 0x25, 0x38d89c7c, 0x7c, 0x2eeb97cae2a265ff, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc02b217f20, 0xc1ff1a0420, 0x25, 0x38d89c7c, 0x7c, 0x2bd18c0, 0xc20b2907e0, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0f83a0800, 0xc1ff1a0420, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x5000104, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0f83a0800, 0xc1ff1a0420, 0x25, 0x38d89c7c, 0x7c, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0f83a0800, 0xc202372360, 0x20, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0f83a0800, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc02b217ef0, 0xc0bcb563f0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10850825 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec2a0b010, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0d5231c18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0d5231c00, 0xc0c96f8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0d5231c00, 0xc0c96f8000, 0x8000, 0x8000, 0x20301b, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02c9e64c8, 0xc0c96f8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc13c184000, 0xc0bb26ac78, 0x9, 0x9, 0xc000423c00, 0x1007f57aae85f10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc13c184000, 0xc0bb26ac78, 0x9, 0x9, 0x9, 0x601742c6, 0xc1525e0dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0bb26ac78, 0x9, 0x9, 0x2b41140, 0xc13c184000, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0bb26ac40, 0xc1baf20620, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc162e24a80, 0xc144b09bf0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc162e24a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc162e24a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10885463 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed4a21cc0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc19d6db898, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc19d6db880, 0xc29dc0c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc19d6db880, 0xc29dc0c000, 0x8000, 0x8000, 0x20302d, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc005010310, 0xc29dc0c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc18c6e1b60, 0xc0d634cd58, 0x9, 0x9, 0xc00035f500, 0x1007f57aae8fa30, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc18c6e1b60, 0xc0d634cd58, 0x9, 0x9, 0x9, 0x601742de, 0xc16e7d4dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0d634cd58, 0x9, 0x9, 0x2b41140, 0xc18c6e1b60, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0d634cd20, 0xc13de37660, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1656a8480, 0xc1b6bb0270, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1656a8480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1656a8480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10792069 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1e83cf3b0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc157e880c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc133bf6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10895877 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec2adbc38, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00bd7da98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00bd7da80, 0xc084008000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00bd7da80, 0xc084008000, 0x8000, 0x8000, 0x203041, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0042e4220, 0xc084008000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc148086300, 0xc0f7fb3378, 0x9, 0x9, 0xc002c80a80, 0x1007f57aae8f368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc148086300, 0xc0f7fb3378, 0x9, 0x9, 0x9, 0x601742d5, 0xc076257dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f7fb3378, 0x9, 0x9, 0x2b41140, 0xc148086300, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f7fb3340, 0xc20f6b1020, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc165843500, 0xc018f46bd0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc165843500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc165843500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10815272 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4eca5cdb30, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc32c50e818, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc32c50e800, 0xc170f91000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc32c50e800, 0xc170f91000, 0x1000, 0x1000, 0x4db33a, 0xe7c7523f48be, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc25cf06080, 0xc170f91000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc15d8768a0, 0xc170f91000, 0x1000, 0x1000, 0xbffdeeb36df69727, 0x38a56d8cfc1, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc1a1cb5860) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1a1cb5860, 0x4, 0x38a56d8cfc1, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc2acd42140, 0x2b99c80, 0xc19f650880) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 10895875 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc248fcf090, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc13363d9e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc165843500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10941464 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc065ce6480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc129c7eb40, 0xc20cbb2d20, 0x25, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc129c7eb40, 0xc20cbb2d20, 0x25, 0x327bf044, 0x64, 0x8306984ba046e66, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc129c7eb40, 0xc20cbb2d20, 0x25, 0x327bf044, 0x64, 0x2bd18c0, 0xc111440360, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc1ed2d2100, 0xc20cbb2d20, 0x25, 0x327bf044, 0x64, 0x0, 0x6000103, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc1ed2d2100, 0xc20cbb2d20, 0x25, 0x327bf044, 0x64, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc1ed2d2100, 0xc1ec6c2aa0, 0x20, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc1ed2d2100, 0x327bf044, 0x64, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc23dae40c0, 0xc22be97200) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10847492 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e1ca7a88, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc007bdfc98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc007bdfc80, 0xc0b3024000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc007bdfc80, 0xc0b3024000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000000) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0ce99ab30, 0xc0b3024000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc2ba1cd560, 0xc06b81fdf8, 0x9, 0x9, 0xc08f860d40, 0x457243, 0xe7bcfc8ec47c) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc2ba1cd560, 0xc06b81fdf8, 0x9, 0x9, 0x9, 0xbffdee919d83181c, 0x36ad7c4cac4, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc06b81fdf8, 0x9, 0x9, 0x2b41140, 0xc2ba1cd560, 0x0, 0x0, 0xc1baf0e680, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc06b81fdc0, 0xc1baf0e680, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc14a946000, 0xc15ecaa780, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc14a946000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc14a946000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942851 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc1db0a2120, 0xa, 0x2b99c80, 0xc0a8e9b300, 0xc0b53e8450) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10942902 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0aaa58fc0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10935181 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0e1c1bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10895789 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed32a5288, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1d9233398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1d9233380, 0xc081cea000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1d9233380, 0xc081cea000, 0x8000, 0x8000, 0x203041, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc802204420, 0xc081cea000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc142874d80, 0xc07ba5bb58, 0x9, 0x9, 0xc002c00380, 0x1007f57aae8f368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc142874d80, 0xc07ba5bb58, 0x9, 0x9, 0x9, 0x601742d5, 0xc002187dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc07ba5bb58, 0x9, 0x9, 0x2b41140, 0xc142874d80, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc07ba5bb20, 0xc20f6b1040, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc157d7d200, 0xc1251b3a40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc157d7d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc157d7d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10943025 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ef5962528, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fa367e18, 0x72, 0x1000, 0x10fb, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fa367e00, 0xc0812db900, 0x10fb, 0x10fb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fa367e00, 0xc0812db900, 0x10fb, 0x10fb, 0x203021, 0xc0fccdf1a0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc54fd7a0b0, 0xc0812db900, 0x10fb, 0x10fb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc086a43020, 0xc0812db900, 0x10fb, 0x10fb, 0x46032be496003613, 0x7aca2167b772b92, 0xc0997e0900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc121a1ecd8, 0x2b41380, 0xc086a43020, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc121a1ea80, 0x2b48520, 0xc54fd7a0b0, 0x5, 0xc54fd7a0b0, 0xc0f761d5f6) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc121a1ea80, 0x0, 0x0, 0x406) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc121a1ea80, 0xc17684a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc02346d0e0, 0xc17684a000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1a1673a40) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1a1673a40, 0x1, 0xc1a4705f80, 0xc0be237cb8, 0x650bd3, 0xc089aa96b0, 0x216f320) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc02346d0e0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 5013409 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc199df4910, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc173fc0660, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1cb237980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10885556 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc163466000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10901773 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc0a2fc3540, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0096b79c0, 0xc23d52b800, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0096b79c0, 0xc095e5e600, 0x2, 0x2, 0xc10d927ce0, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0096b79c0, 0xc120932060, 0xc095e5e600, 0x2, 0x2, 0xc1d56ebc80, 0x2b44f80, 0xc1984dae40, 0x3731625f61663334, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x6f686e69626f725f, 0x72617765725f646f) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1984dae10, 0xc082796230) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10938627 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0d965a7e0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 9835853 [select]: | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc087e46701, 0x1050101) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:391 +0x2ee | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).SyncBlocks.func1(0x2b99c80, 0xc000fa0080, 0xc0005f9e60, 0x0, 0x105b790) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:139 +0x3f | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks.func2(0xc1b0644190, 0xc1a072ec60, 0x26b38d8, 0x2b99c80, 0xc000fa0080, 0xc1b06441b0, 0xc058df7b40) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:186 +0x107 | |
| created by github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).syncUsersBlocks | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:182 +0x45d | |
| goroutine 10896902 [semacquire]: | |
| sync.runtime_Semacquire(0xc00c808070) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc00c808068) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc00c808060, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc089aa93e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1841cfe30, 0x255e040, 0x18, 0xc0b86c73f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc30861e600, 0x2bbccc0, 0xc0465839c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc30861e600, 0x7f57841eead0, 0xc06f4a84a0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc30861e600, 0x2bbbd60, 0xc06f4a84a0, 0xc00083ae00, 0xc0317ed688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc046583920, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc046583900, 0xc0465837a0, 0x26b3908, 0xc046583900, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc046583900, 0x2b99d40, 0xc1841cfce0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0465838e0, 0xc0465837a0, 0xc0465837c0, 0x7f57aae60080, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0465838e0, 0xc0465837a0, 0xc0465837c0, 0x2b99d01, 0xc0465838e0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0465838e0, 0x2b99d40, 0xc1841cfcb0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0xc0465837a0, 0xc0465837e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0x1, 0xc046583760) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0xc0465837a0, 0xc046583800, 0x2a215445, 0xe7bee5eafb9b) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0x203011, 0x7f57840b44d8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0xc0465837a0, 0xc046583820, 0xc0465837a0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0xc0017b8c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ee40, 0xc0465837a0, 0x26b3908, 0x2b99d40, 0xc1841cee10) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc136b5de00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc136b5de00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc09ce7f0e0, 0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc136b5de00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10942885 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0b005d320) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10026404 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc079279080, 0xc1ae084000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc009230660, 0xc1ae084000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0b25f36e0, 0xc1ae084000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0f449af50, 0x7f56e2d6a100, 0xc0b25f36e0, 0xc1ae084000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0f449af50, 0x7f56e2d6a100, 0xc0b25f36e0, 0xc1ae084000, 0x8000, 0x8000, 0x2b6fc80, 0xc0b25f36e0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc10314e1c0, 0x2b44ea0, 0xc0f449af50, 0xc1ae084000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xeccc1453469b7601, 0x981d63b56cfd4e54, 0xc074147d80, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xeccc1453469b7601, 0x981d63b56cfd4e54, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xeccc1453469b7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc004ed5300, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10850824 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc162e24a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942551 [semacquire]: | |
| sync.runtime_Semacquire(0xc1afc79660) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1afc79658) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1afc79650, 0xc0cda27300, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).fetchMissingSubranges(0xc0005c2720, 0x2b99c80, 0xc0fe116d40, 0xc221997320, 0x2d, 0x683f80, 0x68bc80, 0xc0b4452060, 0xc1afc795c0, 0x687e00, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:464 +0x465 | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).cachedGetRange(0xc0005c2720, 0x2b99c80, 0xc0fe116d40, 0xc221997320, 0x2d, 0x6863a8, 0x4025, 0x2534877, 0x6, 0xc0002ff840, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:380 +0x948 | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0fe116d40, 0xc221997320, 0x2d, 0x6863a8, 0x4025, 0x1930902, 0xc0014849f0, 0x1fbbc00, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:276 +0x195 | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0fe116d40, 0xc00bb0f260, 0x28, 0x6863a8, 0x4025, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readChunkRange(0xc00bb0b740, 0x2b99c80, 0xc0fe116d40, 0x0, 0x6863a8, 0x4025, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1470 +0x256 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketChunkReader).loadChunks(0xc1bbec55c0, 0x2b99c80, 0xc0fe116d40, 0xc1b1b27520, 0x3, 0x4, 0x0, 0x68a3cd006863a8, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:2253 +0xde | |
| github.com/thanos-io/thanos/pkg/store.(*bucketChunkReader).preload.func3(0x0, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:2241 +0x9f | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0fccdefc0, 0xc1d5aa9320) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10897061 [semacquire]: | |
| sync.runtime_Semacquire(0xc1170d9b40) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1170d9b38) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1170d9b30, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc14f9bd560) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1170d99b0, 0x255e040, 0x18, 0xc0b4fd13f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc16e0ef4a0, 0x2bbccc0, 0xc14a1c4b40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc16e0ef4a0, 0x7f57841eead0, 0xc00082a250, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc16e0ef4a0, 0x2bbbd60, 0xc00082a250, 0xc00083ae00, 0xc08e435688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc14a1c4aa0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc14a1c4a80, 0xc14a1c4920, 0x26b3908, 0xc14a1c4a80, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc14a1c4a80, 0x2b99d40, 0xc1170d9860) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc14a1c4a60, 0xc14a1c4920, 0xc14a1c4940, 0x7f57aae77a30, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc14a1c4a60, 0xc14a1c4920, 0xc14a1c4940, 0x2b99d01, 0xc14a1c4a60) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc14a1c4a60, 0x2b99d40, 0xc1170d9830) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0xc14a1c4920, 0xc14a1c4960, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0x1, 0xc14a1c48e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0xc14a1c4920, 0xc14a1c4980, 0x29f7553b, 0xe7bee5c0fc93) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0x203052, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0xc14a1c4920, 0xc14a1c49a0, 0xc14a1c4920, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0xc0ad869c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd6543c0, 0xc14a1c4920, 0x26b3908, 0x2b99d40, 0xc1170d97d0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc03f990500, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc03f990500, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc16ac5fa50, 0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc03f990500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10942752 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc20ce264e0, 0xa, 0x2b99c80, 0xc0a92213c0, 0xc0a7b74c60) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4860025 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01ae7aa50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc277e5cd20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0ba5f7c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 5006017 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4eca5cf570, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fc5f3c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fc5f3c80, 0xc15eda0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fc5f3c80, 0xc15eda0000, 0x8000, 0x8000, 0x203061, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc01a0ee368, 0xc15eda0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc02aa0fe60, 0xc02b206578, 0x9, 0x9, 0xc07ba1a000, 0x1007f57aae5e560, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc02aa0fe60, 0xc02b206578, 0x9, 0x9, 0x9, 0x601742d6, 0xc02dc20dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc02b206578, 0x9, 0x9, 0x2b41140, 0xc02aa0fe60, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02b206540, 0xc20111dc60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc02dbc2000, 0xc1567fa510, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc02dbc2000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc02dbc2000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10885462 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1656a8480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10939915 [select]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1dcc3a000, 0xc1dcc3a060, 0xc1dcc3a0c0, 0xc0dd845b10, 0xc100c7a210, 0xc0dec04dc0, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10894851 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15db39770, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc10fb25b60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc165d4d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10899667 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0xc08e9e7000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc1ac1d90a0, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0ef70b0c0, 0xc23c884000, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0ef70b0c0, 0xc0093c9810, 0x2, 0x2, 0xc16f7b8000, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0ef70b0c0, 0xc11c75ede0, 0xc0093c9810, 0x2, 0x2, 0xc16f7b86c0, 0x2b44f80, 0xc14c7a8180, 0xc32875431085fc7f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xd00ea3674248b, 0x5705200640edbf0a) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc14c7a8150, 0xc03030f420) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10938474 [select]: | |
| net/http.(*persistConn).readLoop(0xc0dc4ce6c0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10026062 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc181600000, 0xc1f6636000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc15c2a2480, 0xc1f6636000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0d8ba40f0, 0xc1f6636000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0d5eb0780, 0x7f56e2d6a100, 0xc0d8ba40f0, 0xc1f6636000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0d5eb0780, 0x7f56e2d6a100, 0xc0d8ba40f0, 0xc1f6636000, 0x8000, 0x8000, 0x2b6fc80, 0xc0d8ba40f0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0fd0b8c00, 0x2b44ea0, 0xc0d5eb0780, 0xc1f6636000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x4ff2106ddd3f7601, 0x187e51a9ba903d94, 0xc19ee213c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x4ff2106ddd3f7601, 0x187e51a9ba903d94, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x4ff2106ddd3f7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0028a9800, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10813740 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e1ca7ee8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc10f9d4318, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc10f9d4300, 0xc47eb04000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc10f9d4300, 0xc47eb04000, 0x1000, 0x1000, 0x4378cc, 0xc14ebafb58, 0x4647d0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc15fd0a0b8, 0xc47eb04000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*persistConn).Read(0xc1495d6120, 0xc47eb04000, 0x1000, 0x1000, 0xc017b3e960, 0xc14ebafc58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc0e72e4300) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc0e72e4300, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc1bf38ad20) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc1495d6120) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10898499 [semacquire]: | |
| sync.runtime_Semacquire(0xc15bd00880) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc15bd00878) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc15bd00870, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1551ee900) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc15bd006f0, 0x255e040, 0x18, 0xc030f593f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc14642df80, 0x2bbccc0, 0xc02af9aaa0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc14642df80, 0x7f57841eead0, 0xc0066b9f90, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc14642df80, 0x2bbbd60, 0xc0066b9f90, 0xc00083ae00, 0xc087e1d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc02af9aa00, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc02af9a9e0, 0xc02af9a880, 0x26b3908, 0xc02af9a9e0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc02af9a9e0, 0x2b99d40, 0xc15bd00570) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02af9a9c0, 0xc02af9a880, 0xc02af9a8a0, 0x7f57aae7eca0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02af9a9c0, 0xc02af9a880, 0xc02af9a8a0, 0x2b99d01, 0xc02af9a9c0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02af9a9c0, 0x2b99d40, 0xc15bd00540) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0xc02af9a880, 0xc02af9a8c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0x1, 0xc02af9a840) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0xc02af9a880, 0xc02af9a8e0, 0x3986e0d1, 0xe7beb9b5be31) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0x20300a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0xc02af9a880, 0xc02af9a900, 0xc02af9a880, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0xc14cf31c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a798240, 0xc02af9a880, 0x26b3908, 0x2b99d40, 0xc15bd004e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc153326480, 0xc13a103e00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc153326480, 0xc13a103e00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0d76d0720, 0xc000c0cea0, 0x2bc6d40, 0xc153326480, 0xc13a103e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 5005916 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec5eca008, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1029e8e98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1029e8e80, 0xc15ecc8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1029e8e80, 0xc15ecc8000, 0x8000, 0x8000, 0x800010601, 0x7f4e00000000, 0x1060147e95cc0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc005ae4358, 0xc15ecc8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1341ceae0, 0xc085ad8ab8, 0x9, 0x9, 0xc047e95d40, 0x457243, 0xe7bcf5451073) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1341ceae0, 0xc085ad8ab8, 0x9, 0x9, 0x9, 0xbffdee91964765b7, 0x36ad0891851, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc085ad8ab8, 0x9, 0x9, 0x2b41140, 0xc1341ceae0, 0x0, 0x0, 0xc1c111eb40, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc085ad8a80, 0xc1c111eb40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc032a28300, 0xc160056450, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc032a28300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc032a28300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10936301 [select]: | |
| net/http.(*persistConn).readLoop(0xc100372240) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10941337 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ee9be2da0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f8dfe498, 0x72, 0xdb00, 0xdb99, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f8dfe480, 0xc26bd06000, 0xdb99, 0xdb99, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f8dfe480, 0xc26bd06000, 0xdb99, 0xdb99, 0x203020, 0x752c50, 0xc0e14ff2b8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0f21a34d0, 0xc26bd06000, 0xdb99, 0xdb99, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc080f01ec0, 0xc26bd06000, 0xdb99, 0xdb99, 0x2a28, 0x9752, 0xc1f5a06e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e14ff3d8, 0x2b41380, 0xc080f01ec0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e14ff180, 0x2b48520, 0xc0f21a34d0, 0x5, 0xc0f21a34d0, 0x2a18) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e14ff180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e14ff180, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0e2666d80, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x1a18, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc0cf00f620, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x1a18, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc16a680360, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x1a18, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0f7dfe0c0, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x1a18, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0f7dfe0c0, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0f7dfe100, 0xc1f1d0880a, 0x57f6, 0x57f6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0f7dfe100, 0xc1f1d06000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1db836000, 0xc1db836060, 0xc1db8360c0, 0xc0093a1800, 0xc102a18090, 0xc0d134a2c0, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10886712 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1064c9380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4861635 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc079c43090, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc271157620, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1b9453e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10850823 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02b1a2e10, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc146021800, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc162e24a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4975788 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0f52729b0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1a76a85a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc07988d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4860164 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0aeaedb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10896904 [semacquire]: | |
| sync.runtime_Semacquire(0xc009222e50) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc009222e48) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc009222e40, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc16532c780) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc009222c60, 0x255e040, 0x18, 0xc0f381d3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1117d3bc0, 0x2bbccc0, 0xc0da01cfc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1117d3bc0, 0x7f57841eead0, 0xc16ff48f30, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1117d3bc0, 0x2bbbd60, 0xc16ff48f30, 0xc00083ae00, 0xc08989f688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc0da01cf20, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc0da01cf00, 0xc0da01cda0, 0x26b3908, 0xc0da01cf00, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc0da01cf00, 0x2b99d40, 0xc009222a50) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0da01cee0, 0xc0da01cda0, 0xc0da01cdc0, 0x7f57aae60080, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0da01cee0, 0xc0da01cda0, 0xc0da01cdc0, 0x2b99d01, 0xc0da01cee0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0da01cee0, 0x2b99d40, 0xc009222a20) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0xc0da01cda0, 0xc0da01cde0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0x1, 0xc0da01cd60) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0xc0da01cda0, 0xc0da01ce00, 0x29f6b2bc, 0xe7bee5c05a14) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0x203036, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0xc0da01cda0, 0xc0da01ce20, 0xc0da01cda0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0xc0017b8c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3e180, 0xc0da01cda0, 0x26b3908, 0x2b99d40, 0xc0092229c0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc13872e000, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc13872e000, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc09ce7f0e0, 0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc13872e000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10938571 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0df5ebc20) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10943009 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783d33e40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0cfabba98, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0cfabba80, 0xc228feb900, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0cfabba80, 0xc228feb900, 0x109b, 0x109b, 0x20304d, 0xc0a7b749f0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc8457720a0, 0xc228feb900, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc1370746a0, 0xc228feb900, 0x109b, 0x109b, 0x1, 0x1, 0xc06aed6900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0d4d113d8, 0x2b41380, 0xc1370746a0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0d4d11180, 0x2b48520, 0xc8457720a0, 0x5, 0xc8457720a0, 0x4daf20) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0d4d11180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0d4d11180, 0xc175f14000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0ae1d9680, 0xc175f14000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc0e59a7ec0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc0e59a7ec0, 0x1, 0xc20ce269c0, 0xc0f9a5ae00, 0x412fae, 0xbebc200, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0ae1d9680) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10026064 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc0e1516c00, 0xc1f1a12000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc13d678660, 0xc1f1a12000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc102a18300, 0xc1f1a12000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0fd6650e0, 0x7f56e2d6a100, 0xc102a18300, 0xc1f1a12000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0fd6650e0, 0x7f56e2d6a100, 0xc102a18300, 0xc1f1a12000, 0x8000, 0x8000, 0x2b6fc80, 0xc102a18300, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1007aab00, 0x2b44ea0, 0xc0fd6650e0, 0xc1f1a12000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x3b92717d2e2a7701, 0x5b7cd05955db780b, 0xc16cc5b2c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x3b92717d2e2a7701, 0x5b7cd05955db780b, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x3b92717d2e2a7701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0016c6e00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10847636 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e1ca78c8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc803d0c818, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc803d0c800, 0xc0b6dda000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc803d0c800, 0xc0b6dda000, 0x8000, 0x8000, 0x203063, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1a55fa6b8, 0xc0b6dda000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc168c033e0, 0xc0736fded8, 0x9, 0x9, 0xc071352a80, 0x1007f57aae5ec28, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc168c033e0, 0xc0736fded8, 0x9, 0x9, 0x9, 0x601742e4, 0xc13e7eedc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0736fded8, 0x9, 0x9, 0x2b41140, 0xc168c033e0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0736fdea0, 0xc0f525d400, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc125006d80, 0xc115a2ba40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc125006d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc125006d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10777742 [select, 2 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc21722af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10941542 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f31f7b00) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10942585 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e2cc8900, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fa0a0118, 0x72, 0xc900, 0xc935, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fa0a0100, 0xc26bdc10c8, 0xc935, 0xc935, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fa0a0100, 0xc26bdc10c8, 0xc935, 0xc935, 0x20306d, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0dd632810, 0xc26bdc10c8, 0xc935, 0xc935, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc1b72283a0, 0xc26bdc10c8, 0xc935, 0xc935, 0x10c8, 0x2b415c0, 0xc02325ee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0d9b7fad8, 0x2b41380, 0xc1b72283a0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0d9b7f880, 0x2b48520, 0xc0dd632810, 0x15b5, 0xc0dd632810, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0d9b7f880, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:634 +0x573 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0d9b7f880, 0xc174b1e8e8, 0x1598, 0x1598, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0ca67d200, 0xc174b1e8e8, 0x1598, 0x1598, 0x2328, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc0fdcf8b40, 0xc174b1e8e8, 0x1598, 0x1598, 0x2328, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc087dc5d40, 0xc174b1e8e8, 0x1598, 0x1598, 0x2328, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0aa529dc0, 0xc174b1e8e8, 0x1598, 0x1598, 0x2328, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0aa529dc0, 0xc174b1e8e8, 0x1598, 0x1598, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0aa529e00, 0xc174b1e8e8, 0x1598, 0x1598, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0aa529e00, 0xc174b1c000, 0x3e80, 0x3e80, 0x3e80, 0x3e80, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc103ab9260, 0xc103ab92c0, 0xc103ab9380, 0xc16cee6780, 0xc1afc796e0, 0xc0b72d2bb0, 0x2b99d40, 0xc1afc79680, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10902202 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x0, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc190012780, 0x15, 0x7f3e16fe79b7, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0045239c0, 0xc0fc3feb00, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0045239c0, 0xc009460e20, 0x2, 0x2, 0xc17ee1b170, 0xd, 0x1, 0xc002d80380, 0x7f57aae5d7d0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc0045239c0, 0xc1c03715c0, 0xc009460e20, 0x2, 0x2, 0xc009e8bb60, 0x2b44f80, 0xc14f9bdaa0, 0x26, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc14f9bda70, 0xc08280e230) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10940913 [semacquire]: | |
| sync.runtime_Semacquire(0xc1022407c0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1022407b8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1022407b0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1028d4060) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc102240630, 0x255e040, 0x18, 0xc032c833f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1a7284de0, 0x2bbccc0, 0xc094470700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1a7284de0, 0x7f57841eead0, 0xc0d5e13890, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1a7284de0, 0x2bbbd60, 0xc0d5e13890, 0xc00083ae00, 0xc08e839688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc094470660, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc094470640, 0xc0944704e0, 0x26b3908, 0xc094470640, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc094470640, 0x2b99d40, 0xc1022404e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc094470620, 0xc0944704e0, 0xc094470500, 0x7f57aae8a5d8, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc094470620, 0xc0944704e0, 0xc094470500, 0x2b99d01, 0xc094470620) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc094470620, 0x2b99d40, 0xc1022404b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0xc0944704e0, 0xc094470520, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0x1, 0xc0944704a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0xc0944704e0, 0xc094470540, 0x2bdc241d, 0xe7c38fbd9373) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0x203025, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0xc0944704e0, 0xc094470560, 0xc0944704e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0xc16e5a2c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc162292180, 0xc0944704e0, 0x26b3908, 0x2b99d40, 0xc102240450) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc0d99a2a00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc0d99a2a00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc16ac5fa50, 0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc0d99a2a00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10942881 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0f88f2b40, 0xc0a49e97c0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0d934b200, 0xc17b484000, 0xc0aff54678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0d934b200, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0d934b200, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc182bf0158, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0d934b200, 0x0, 0x0, 0x0, 0xc182bf0158, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0d934b200, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc210c365a0, 0xc210c36600, 0xc210c36660, 0xc0b71b7ee0, 0xc0b50e7230, 0xc0ccfa7080, 0x2b99d40, 0xc0b50e71d0, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10939115 [select]: | |
| net/http.(*persistConn).readLoop(0xc1003725a0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10900998 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x3f74400, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc1f83bd160, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00582b900, 0xc0cc896240, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00582b900, 0xc00be2d960, 0x2, 0x2, 0xc120933c80, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc00582b900, 0xc1d56ea360, 0xc00be2d960, 0x2, 0x2, 0xc1d56ea000, 0x2b44f80, 0xc144a96090, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc144a96060, 0xc081547f10) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5005914 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1a77078b0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0220e99e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc032a28300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10884841 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc028320300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942850 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0aaa58fc0, 0xc0a8e9b400, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0d3909300, 0xc1e2c674a0, 0xc02e3fe678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0d3909300, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0d3909300, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc165d3e098, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0d3909300, 0x0, 0x0, 0x0, 0xc165d3e098, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0d3909300, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc103473f80, 0xc1db0a2000, 0xc1db0a2060, 0xc0b69aa810, 0xc0b53e83c0, 0xc0b923ab00, 0x2b99d40, 0xc0b53e8360, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10942884 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e2d2aa88, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fc9f7798, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fc9f7780, 0xc228fea600, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fc9f7780, 0xc228fea600, 0x109b, 0x109b, 0x203045, 0xc0997de850, 0x417a5e) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0e5a3e0f8, 0xc228fea600, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc115c5a120, 0xc228fea600, 0x109b, 0x109b, 0xafc979c143d9195e, 0x60da716e23b30d78, 0xc0997de900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0b21a1e58, 0x2b41380, 0xc115c5a120, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0b21a1c00, 0x2b48520, 0xc0e5a3e0f8, 0x5, 0xc0e5a3e0f8, 0xc0fd4e43f2) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0b21a1c00, 0x0, 0x0, 0x4cbffa) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0b21a1c00, 0xc176744000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0b005d320, 0xc176744000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc17b484960) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc17b484960, 0x1, 0xc20a7f4b40, 0x0, 0x7701000000000091, 0x1268a21099145f4d, 0x3130b68face94e8e) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0b005d320) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10939424 [runnable]: | |
| sync.(*Mutex).Unlock(0xc0e2344838) | |
| /usr/local/go/src/sync/mutex.go:179 +0x4f | |
| crypto/tls.(*Conn).Read(0xc0e2344700, 0xc1f1a149e2, 0x561e, 0x561e, 0x4000, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1259 +0x243 | |
| net/http.(*persistConn).Read(0xc0f8ec2120, 0xc1f1a149e2, 0x561e, 0x561e, 0x650, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc096d8d8c0, 0xc1f1a149e2, 0x561e, 0x561e, 0x650, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc219f2e040, 0xc1f1a149e2, 0x561e, 0x561e, 0x650, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0fe7a0000, 0xc1f1a149e2, 0x561e, 0x561e, 0x650, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0fe7a0000, 0xc1f1a149e2, 0x561e, 0x561e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0fe7a0040, 0xc1f1a149e2, 0x561e, 0x561e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0fe7a0040, 0xc1f1a12000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0e8a22000, 0xc0e8a22060, 0xc0e8a220c0, 0xc1038dc5f0, 0xc0fff14090, 0xc0fa3f1c30, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10894853 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4eca5cdcf0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc15ba50618, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc15ba50600, 0xc085630000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc15ba50600, 0xc085630000, 0x8000, 0x8000, 0x203022, 0x0, 0x601132a4cc0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0e5a3e1a8, 0xc085630000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc145f101e0, 0xc0f305dd18, 0x9, 0x9, 0xc000423500, 0x1007f57aae900f8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc145f101e0, 0xc0f305dd18, 0x9, 0x9, 0x9, 0x601742d1, 0xc1132a4dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f305dd18, 0x9, 0x9, 0x2b41140, 0xc145f101e0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f305dce0, 0xc2109627e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc165d4d500, 0xc117d8d1a0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc165d4d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc165d4d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10938455 [select]: | |
| net/http.(*persistConn).readLoop(0xc0e2666d80) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10942792 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc208a8a360, 0xa, 0x2b99c80, 0xc1546c6740, 0xc129c7e5d0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10941465 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1e3884a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc23dae40f0, 0xc1f31c1a40, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc23dae40f0, 0xc1f31c1a40, 0x25, 0x38d89c7c, 0x7c, 0x8306984ba046e66, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc23dae40f0, 0xc1f31c1a40, 0x25, 0x38d89c7c, 0x7c, 0x2bd18c0, 0xc02346cea0, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc1ed2d2100, 0xc1f31c1a40, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x4000105, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc1ed2d2100, 0xc1f31c1a40, 0x25, 0x38d89c7c, 0x7c, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc1ed2d2100, 0xc20216a300, 0x20, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc1ed2d2100, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0xc0cc4b1180, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc23dae40c0, 0xc22be97290) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10895876 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc165843500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942589 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0b005d320, 0xc0a9ee7900, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0ad882e00, 0xc1792d40c0, 0xc0cf714678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0ad882e00, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0ad882e00, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc845772080, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0ad882e00, 0x0, 0x0, 0x0, 0xc845772080, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0ad882e00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc103ab98c0, 0xc103ab9920, 0xc103ab99e0, 0xc16cee6cb0, 0xc1afc79e30, 0xc0b72d2fd0, 0x2b99d40, 0xc1afc79dd0, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10942829 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc13cc2a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0b3a16480, 0xc222403680, 0x25, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0b3a16480, 0xc222403680, 0x25, 0x327bf044, 0x64, 0x4bf892b02a805d42, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0b3a16480, 0xc222403680, 0x25, 0x327bf044, 0x64, 0x2bd18c0, 0xc10c16a5a0, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0fa907d80, 0xc222403680, 0x25, 0x327bf044, 0x64, 0x0, 0x1000100, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0fa907d80, 0xc222403680, 0x25, 0x327bf044, 0x64, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0fa907d80, 0xc1eafaf720, 0x20, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0fa907d80, 0x327bf044, 0x64, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0b31d7e00, 0xc0dc5606c0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10898137 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02b5f7040, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc11f72af60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc165902f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10902506 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0xc0317af000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc18e520540, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1b4616a80, 0xc0dd6546c0, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1b4616a80, 0xc16aa5b270, 0x2, 0x2, 0xc00bb03da0, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc1b4616a80, 0xc101247bc0, 0xc16aa5b270, 0x2, 0x2, 0xc00bb03e00, 0x2b44f80, 0xc1a121a180, 0x128b9100, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x128bab40, 0x128baba0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1a121a150, 0xc08276b730) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10792070 [select, 2 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc133bf6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10893274 [semacquire]: | |
| sync.runtime_Semacquire(0xc06fffcaf0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc06fffcae8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc06fffcae0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc02b22ecf0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc06fffc960, 0x255e040, 0x18, 0xc0f9edd3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc0080a1c80, 0x2bbccc0, 0xc1855a3a40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc0080a1c80, 0x7f57841eead0, 0xc16cee6ee0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc0080a1c80, 0x2bbbd60, 0xc16cee6ee0, 0xc00083ae00, 0xc1051db688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc1855a39a0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc1855a3980, 0xc1855a3820, 0x26b3908, 0xc1855a3980, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc1855a3980, 0x2b99d40, 0xc06fffc7e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc1855a3960, 0xc1855a3820, 0xc1855a3840, 0x7f57aae71848, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc1855a3960, 0xc1855a3820, 0xc1855a3840, 0x2b99d01, 0xc1855a3960) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc1855a3960, 0x2b99d40, 0xc06fffc7b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0xc1855a3820, 0xc1855a3860, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0x1, 0xc1855a37e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0xc1855a3820, 0xc1855a3880, 0x2a0a84b7, 0xe7bee5d42c0f) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0x203061, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0xc1855a3820, 0xc1855a38a0, 0xc1855a3820, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0xc12c1aec68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382780, 0xc1855a3820, 0x26b3908, 0x2b99d40, 0xc06fffc750) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780, 0xc136aba000, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780, 0xc136aba000, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc07f479550, 0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780, 0xc136aba000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 5013411 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec2adb618, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc16e925998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc16e925980, 0xc1b51c0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc16e925980, 0xc1b51c0000, 0x8000, 0x8000, 0x20301f, 0x7f4e00000000, 0x601aa967cc0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02b739098, 0xc1b51c0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc176b46840, 0xc0664242d8, 0x9, 0x9, 0xc0b02e5880, 0x1007f57aae89f10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc176b46840, 0xc0664242d8, 0x9, 0x9, 0x9, 0x601742d5, 0xc2aa967dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0664242d8, 0x9, 0x9, 0x2b41140, 0xc176b46840, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0664242a0, 0xc14d616000, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1cb237980, 0xc1760e9080, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1cb237980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1cb237980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10792065 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1e83cf130, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc111f8ff20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc133bf6000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4859012 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1b9453380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10901432 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc0cd4b4b80, 0x15, 0x7f3e16fe7905, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc004ed9ec0, 0xc0f9e71300, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc004ed9ec0, 0xc009460240, 0x2, 0x2, 0x7b0053e0b608b438, 0x54200be9c6d0f22a, 0x10e674a3c6f795e, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc004ed9ec0, 0xc009e8afc0, 0xc009460240, 0x2, 0x2, 0xc16e0ef4a0, 0x2b44f80, 0xc1170d9b60, 0x6f6f622e63696c62, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x6e635f736e632e63, 0x6f697469736f7073) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1170d9b30, 0xc080fd52d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10935182 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4eca5cf2d0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0e652c798, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0e652c780, 0xc174908000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0e652c780, 0xc174908000, 0x8000, 0x8000, 0x203052, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc165d3ea68, 0xc174908000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1b4609aa0, 0xc03b89c038, 0x9, 0x9, 0xc000581180, 0x1007f57aae7fa30, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1b4609aa0, 0xc03b89c038, 0x9, 0x9, 0x9, 0x601742d9, 0xc10f288dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc03b89c038, 0x9, 0x9, 0x2b41140, 0xc1b4609aa0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc03b89c000, 0xc1fb3b83c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0e1c1bb00, 0xc0f45a2a80, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0e1c1bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0e1c1bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4861132 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15db39630, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc208a12c00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1b9453680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10847491 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc14a946000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10884842 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e151ad10, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1af554c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1af554c80, 0xc17c94c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1af554c80, 0xc17c94c000, 0x8000, 0x8000, 0x10401, 0x0, 0x104010020302f) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc122508350, 0xc17c94c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1b6424840, 0xc0e21dc3b8, 0x9, 0x9, 0xc0011f7180, 0x7f57aae725d8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1b6424840, 0xc0e21dc3b8, 0x9, 0x9, 0x9, 0xa6fc25, 0xc0be604000, 0xc0be600004) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0e21dc3b8, 0x9, 0x9, 0x2b41140, 0xc1b6424840, 0x0, 0x0, 0xc0be604000, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0e21dc380, 0xc0be604000, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc028320300, 0xc1b5e5c6f0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc028320300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc028320300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942887 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0adf4f320) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10941818 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56b9ba6548, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1029a6998, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1029a6980, 0xc1029db000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1029a6980, 0xc1029db000, 0x1000, 0x1000, 0x4db33a, 0xe7ca977d9188, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0232567a8, 0xc1029db000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc1029ed2f0, 0xc1029db000, 0x1000, 0x1000, 0xbffdeeb6f6457f82, 0x38da19ec41c, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc1b8b4a420) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1b8b4a420, 0x4, 0x38da19ec41c, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc104090000, 0x2b99c80, 0xc1040801c0) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 10942633 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0f138f3e0, 0xa, 0x2b99c80, 0xc0fd226c80, 0xc0b3a16570) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10902835 [runnable]: | |
| github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).UvarintBytes(0xc0317b2c80, 0x0, 0x2, 0xc09dbfdaa0) | |
| /__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go:192 +0x115 | |
| github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).UvarintStr(...) | |
| /__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go:187 | |
| github.com/prometheus/prometheus/tsdb/index.ReadOffsetTable(0x2b73f00, 0xc037e74020, 0x2fe9f70, 0xc0317b2e28, 0xc1009ead80, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/prometheus/prometheus/tsdb/index/index.go:1407 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.newFileBinaryReader(0xc20122a300, 0x37, 0x20, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:550 +0xa4f | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x304b65decd507701, 0x9c3e91e302eda556, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:451 +0x176 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:234 +0x332 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc0f5a3af00, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0037fba40, 0xc0fed92180, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0037fba40, 0xc16aa5bd00, 0x2, 0x2, 0xc00bb03da0, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0037fba40, 0xc14dd219e0, 0xc16aa5bd00, 0x2, 0x2, 0xc101247c20, 0x2b44f80, 0xc0ac8ae900, 0x4e, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x1da90ea, 0xc18ff9e960) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0ac8ae8d0, 0xc083118230) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10938475 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0dc4ce6c0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10852985 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001485180, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc13da88fc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc25b9ccd80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10938585 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f8ec2120) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10941794 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4f47fba7c8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f9af4018, 0x72, 0xd000, 0xd09d, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f9af4000, 0xc26bcd8000, 0xd09d, 0xd09d, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f9af4000, 0xc26bcd8000, 0xd09d, 0xd09d, 0x203040, 0x752c50, 0xc0d9b7f638) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0dd632808, 0xc26bcd8000, 0xd09d, 0xd09d, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc101fce980, 0xc26bcd8000, 0xd09d, 0xd09d, 0x2a98, 0x8839, 0xc0cdfdee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0d9b7f758, 0x2b41380, 0xc101fce980, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0d9b7f500, 0x2b48520, 0xc0dd632808, 0x5, 0xc0dd632808, 0x2a88) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0d9b7f500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0d9b7f500, 0xc1f5a52822, 0x17de, 0x17de, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0e809d8c0, 0xc1f5a52822, 0x17de, 0x17de, 0x2a88, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc10f547680, 0xc1f5a52822, 0x17de, 0x17de, 0x2a88, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc1668740e0, 0xc1f5a52822, 0x17de, 0x17de, 0x2a88, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc1009ea000, 0xc1f5a52822, 0x17de, 0x17de, 0x2a88, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc1009ea000, 0xc1f5a52822, 0x17de, 0x17de, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc1009ea040, 0xc1f5a52822, 0x17de, 0x17de, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc1009ea040, 0xc1f5a4c000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0da5aa000, 0xc0da5aa060, 0xc0da5aa0c0, 0xc0ef6577c0, 0xc103e1c240, 0xc0e47fcf20, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10943011 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f56e2c40ef0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1127b2898, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1127b2880, 0xc228fe9300, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1127b2880, 0xc228fe9300, 0x109b, 0x109b, 0x203045, 0xc1127b2798, 0xc1127b2780) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1a55fa158, 0xc228fe9300, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc115c5a100, 0xc228fe9300, 0x109b, 0x109b, 0xcb541f64e2885b2c, 0x59f111f13956c25b, 0xc07625a900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc1880bf758, 0x2b41380, 0xc115c5a100, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc1880bf500, 0x2b48520, 0xc1a55fa158, 0x5, 0xc1a55fa158, 0xd5a79147c6e00bf3) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc1880bf500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc1880bf500, 0xc16a55c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f9db57a0, 0xc16a55c000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc2c1818180) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc2c1818180, 0x1, 0xc20a7f4ae0, 0xc0b1a6c4b8, 0x650bd3, 0xc153a0df80, 0x216f320) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0f9db57a0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10941226 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4eca57a900, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f9f9e818, 0x72, 0x13e00, 0x13e17, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f9f9e800, 0xc26bc5a000, 0x13e17, 0x13e17, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f9f9e800, 0xc26bc5a000, 0x13e17, 0x13e17, 0x20303f, 0x752c50, 0xc0e505d638) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0e22f7c50, 0xc26bc5a000, 0x13e17, 0x13e17, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc0fee27be0, 0xc26bc5a000, 0x13e17, 0x13e17, 0x2588, 0xf9d0, 0xc0c60cee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e505d758, 0x2b41380, 0xc0fee27be0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e505d500, 0x2b48520, 0xc0e22f7c50, 0x5, 0xc0e22f7c50, 0x2578) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e505d500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e505d500, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0d965ac60, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x14ca, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc0fdcf87e0, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x14ca, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc0f7bbc860, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x14ca, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0fb0a8200, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x14ca, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0fb0a8200, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0fb0a8240, 0xc1f1d434ca, 0x6b36, 0x6b36, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0fb0a8240, 0xc1f1d42000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc01d51c000, 0xc01d51c060, 0xc01d51c180, 0xc0f6125dd0, 0xc0e27c29f0, 0xc07cd4ed10, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10938616 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0e809d560) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10942632 [select]: | |
| net/http.(*persistConn).roundTrip(0xc10c16ac60, 0xc0fd226d80, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0fc2f8900, 0xc1222919e0, 0xc0b03ee678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0fc2f8900, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0fc2f8900, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc1ec284068, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0fc2f8900, 0x0, 0x0, 0x0, 0xc1ec284068, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0fc2f8900, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0f138f2c0, 0xc0f138f320, 0xc0f138f380, 0xc0fd21b7e0, 0xc0b3a164e0, 0xc030a3b600, 0x2b99d40, 0xc0b3a16480, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10929085 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783db12b0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fd563198, 0x72, 0xe600, 0xe6f2, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fd563180, 0xc26e002000, 0xe6f2, 0xe6f2, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fd563180, 0xc26e002000, 0xe6f2, 0xe6f2, 0x203040, 0x752c50, 0xc0e851af38) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0a983f008, 0xc26e002000, 0xe6f2, 0xe6f2, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc101fb79e0, 0xc26e002000, 0xe6f2, 0xe6f2, 0x1e20, 0x5e71, 0xc0325e8e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e851b058, 0x2b41380, 0xc101fb79e0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e851ae00, 0x2b48520, 0xc0a983f008, 0x5, 0xc0a983f008, 0x1e10) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e851ae00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e851ae00, 0xc1a47e7f72, 0x608e, 0x608e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0fc3b58c0, 0xc1a47e7f72, 0x608e, 0x608e, 0x1e10, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc49b7b23c0, 0xc1a47e7f72, 0x608e, 0x608e, 0x1e10, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc0b8f52900, 0xc1a47e7f72, 0x608e, 0x608e, 0x1e10, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc101b5f440, 0xc1a47e7f72, 0x608e, 0x608e, 0x1e10, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc101b5f440, 0xc1a47e7f72, 0x608e, 0x608e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0d6c13880, 0xc1a47e7f72, 0x608e, 0x608e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0d6c13880, 0xc1a47e6000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc093328000, 0xc093328060, 0xc093328180, 0xc103ec64d0, 0xc0b9892030, 0xc0d7349340, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10853743 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0de205200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10941320 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4edb4860b0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f9973018, 0x72, 0xe500, 0xe523, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f9973000, 0xc26bd14000, 0xe523, 0xe523, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f9973000, 0xc26bd14000, 0xe523, 0xe523, 0x203040, 0x752c50, 0xc0f97404b8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02d43b7e0, 0xc26bd14000, 0xe523, 0xe523, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc101fb7520, 0xc26bd14000, 0xe523, 0xe523, 0x3d68, 0xe516, 0xc0b03eae90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0f97405d8, 0x2b41380, 0xc101fb7520, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0f9740380, 0x2b48520, 0xc02d43b7e0, 0x5, 0xc02d43b7e0, 0x3d58) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0f9740380, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0f9740380, 0xc1ae089ab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0df5ebc20, 0xc1ae089ab2, 0x254e, 0x254e, 0x3d58, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc01836d020, 0xc1ae089ab2, 0x254e, 0x254e, 0x3d58, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc0a6ca5ce0, 0xc1ae089ab2, 0x254e, 0x254e, 0x3d58, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0a49e8380, 0xc1ae089ab2, 0x254e, 0x254e, 0x3d58, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0a49e8380, 0xc1ae089ab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0a49e83c0, 0xc1ae089ab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0a49e83c0, 0xc1ae084000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0eca96000, 0xc0eca96060, 0xc0eca96120, 0xc10037c730, 0xc0fe8246c0, 0xc0e203b4a0, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10899405 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc1baf0fb40, 0x15, 0x7f3e16fe7905, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1b4e17080, 0xc0a53d2900, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1b4e17080, 0xc14ac6f470, 0x2, 0x2, 0xc25c078fd0, 0xd, 0x1, 0xc000423500, 0x7f57aae707c0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc1b4e17080, 0xc00c1ae9c0, 0xc14ac6f470, 0x2, 0x2, 0xc1645fa180, 0x2b44f80, 0xc0f99bdfb0, 0xc00017c0b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0f99bdf80, 0xc0778ab500) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5015548 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1e417f5e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc17acf6d80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1bf470780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10940466 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0fc3b58c0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10026402 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc16638f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99c80, 0xc000fa0080, 0xc1f5b81f20, 0x25, 0x18102316, 0x1f4dda1, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99c80, 0xc000fa0080, 0xc1f5b81f20, 0x25, 0x18102316, 0x1f4dda1, 0xc0b00cb5c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99c80, 0xc000fa0080, 0xc1f5b81f20, 0x25, 0x18102316, 0x1f4dda1, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc000fa0080, 0xc1f5b81f20, 0x25, 0x18102316, 0x1f4dda1, 0x0, 0x1000100, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc000fa0080, 0xc1f5b81f20, 0x25, 0x18102316, 0x1f4dda1, 0x0, 0x7f56e2d6a100, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc000fa0080, 0xc13cccc1a0, 0x20, 0x18102316, 0x1f4dda1, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc088033480, 0x2b44ea0, 0xc0f4d0b0e0, 0xc1a66ae000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x713def6e94127701, 0x40beebc601f61522, 0xc0741468c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x713def6e94127701, 0x40beebc601f61522, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x713def6e94127701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc003183b00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10792071 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec4fc7c48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1381ea918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1381ea900, 0xc17b674000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1381ea900, 0xc17b674000, 0x8000, 0x8000, 0x203096, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc269754708, 0xc17b674000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc111f8fec0, 0xc05bdb23b8, 0x9, 0x9, 0xc065237500, 0x1007f57aae8c7c0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc111f8fec0, 0xc05bdb23b8, 0x9, 0x9, 0x9, 0x601742d5, 0xc16236cdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc05bdb23b8, 0x9, 0x9, 0x2b41140, 0xc111f8fec0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc05bdb2380, 0xc1ef3ae1a0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc133bf6300, 0xc133bec3f0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc133bf6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc133bf6300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10900994 [semacquire]: | |
| sync.runtime_Semacquire(0xc144a96070) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc144a96068) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc144a96060, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc144a97d40) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc18a81de90, 0x255e040, 0x18, 0xc2107c33f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1d56ea000, 0x2bbccc0, 0xc0b02394a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1d56ea000, 0x7f57841eead0, 0xc00be2cd40, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1d56ea000, 0x2bbbd60, 0xc00be2cd40, 0xc00083ae00, 0xc0887a3688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc0b0239400, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc0b02393e0, 0xc0b0239280, 0x26b3908, 0xc0b02393e0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc0b02393e0, 0x2b99d40, 0xc18a81dd40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0b02393c0, 0xc0b0239280, 0xc0b02392a0, 0x7f57aae76ca0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0b02393c0, 0xc0b0239280, 0xc0b02392a0, 0x2b99d01, 0xc0b02393c0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc0b02393c0, 0x2b99d40, 0xc18a81dd10) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0xc0b0239280, 0xc0b02392c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0x1, 0xc0b0239240) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0xc0b0239280, 0xc0b02392e0, 0x29f6d2ba, 0xe7bee5c07a15) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0x20302c, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0xc0b0239280, 0xc0b0239300, 0xc0b0239280, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0xc01a45fc68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6a900, 0xc0b0239280, 0x26b3908, 0x2b99d40, 0xc18a81dcb0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc1dff65380, 0xc079e2ce00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc1dff65380, 0xc079e2ce00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc19ee657d0, 0xc000c0cea0, 0x2bc6d40, 0xc1dff65380, 0xc079e2ce00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4864257 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1d99d5380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4974783 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1b297c190, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc07d44e000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1e3012180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942193 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1a2096000, 0xc1a2096060, 0xc1a20960c0, 0xc0e5709b70, 0xc0f7289170, 0xc0c673d080, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10886260 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1e04b4000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10938303 [select]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1dbad0000, 0xc1dbad0060, 0xc1dbad00c0, 0xc0caf45e40, 0xc0ff936030, 0xc0da25d970, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10898504 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc17eb100c0, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0050ef840, 0xc022ed95c0, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0050ef840, 0xc000c88e00, 0x2, 0x2, 0xc118bba560, 0xd, 0x1, 0xc0dd272e00, 0x7f57aae7df10) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0050ef840, 0xc8166ae300, 0xc000c88e00, 0x2, 0x2, 0xc14642df80, 0x2b44f80, 0xc15bd008a0, 0x1e8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc0700ebfe0, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc15bd00870, 0xc027b18230) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10938508 [select]: | |
| net/http.(*persistConn).readLoop(0xc0e809d8c0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10938778 [select]: | |
| net/http.(*persistConn).readLoop(0xc0faafa360) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10898500 [semacquire]: | |
| sync.runtime_Semacquire(0xc15da39a20) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc15da39a18) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc15da39a10, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc14aa81020) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc10c1bbc20, 0x255e040, 0x18, 0xc044deb3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc14642de60, 0x2bbccc0, 0xc02af9a740, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc14642de60, 0x7f57841eead0, 0xc0066b9e90, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc14642de60, 0x2bbbd60, 0xc0066b9e90, 0xc00083ae00, 0xc087e1d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc02af9a6a0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc02af9a680, 0xc02af9a520, 0x26b3908, 0xc02af9a680, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc02af9a680, 0x2b99d40, 0xc10c1bbaa0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02af9a660, 0xc02af9a520, 0xc02af9a540, 0x7f57aae7eca0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02af9a660, 0xc02af9a520, 0xc02af9a540, 0x2b99d01, 0xc02af9a660) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02af9a660, 0x2b99d40, 0xc10c1bba70) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0xc02af9a520, 0xc02af9a560, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0x1, 0xc02af9a4e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0xc02af9a520, 0xc02af9a580, 0x3389db3f, 0xe7beb3b8b896) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0x20300a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0xc02af9a520, 0xc02af9a5a0, 0xc02af9a520, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0xc14cf31c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a7980c0, 0xc02af9a520, 0x26b3908, 0x2b99d40, 0xc10c1bba10) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc153326480, 0xc13a103f00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc153326480, 0xc13a103f00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0d76d0720, 0xc000c0cea0, 0x2bc6d40, 0xc153326480, 0xc13a103f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4861508 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15bd10a50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc271157260, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc06b2d7500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10026053 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc16639f800, 0xc141d7e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc1345f93e0, 0xc141d7e000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0fb5e3710, 0xc141d7e000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0e6011540, 0x7f56e2d6a100, 0xc0fb5e3710, 0xc141d7e000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0e6011540, 0x7f56e2d6a100, 0xc0fb5e3710, 0xc141d7e000, 0x8000, 0x8000, 0x2b6fc80, 0xc0fb5e3710, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc088032e80, 0x2b44ea0, 0xc0e6011540, 0xc141d7e000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xf824c40d3f57601, 0xe215305d24063fa0, 0xc0628393c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xf824c40d3f57601, 0xe215305d24063fa0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xf824c40d3f57601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc003580d00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10777741 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc17bbfb720, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc130ed5800, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc21722af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 5005856 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc02dbc2000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 5015549 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1bf470780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10861162 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc028ce7b30, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc140a8dec0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0b0442c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10026406 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc22056cd80, 0xc1ae81c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc115e857a0, 0xc1ae81c000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1afc79590, 0xc1ae81c000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0fedd5540, 0x7f56e2d6a100, 0xc1afc79590, 0xc1ae81c000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0fedd5540, 0x7f56e2d6a100, 0xc1afc79590, 0xc1ae81c000, 0x8000, 0x8000, 0x2b6fc80, 0xc1afc79590, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0f2165700, 0x2b44ea0, 0xc0fedd5540, 0xc1ae81c000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x9fe5c831edd67601, 0xf79a1d21857d4706, 0xc1a0bb6e00, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x9fe5c831edd67601, 0xf79a1d21857d4706, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x9fe5c831edd67601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc006354600, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10938456 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0e2666d80) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10941017 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc1a9473ec0, 0x15, 0x7f3e16fe7905, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc102245700, 0xc0b6d20d00, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc102245700, 0xc102732170, 0x2, 0x2, 0xc0d6d53e80, 0xd, 0x1, 0xc000423500, 0x7f57aae8a5d8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc102245700, 0xc111444960, 0xc102732170, 0x2, 0x2, 0xc1a7284de0, 0x2b44f80, 0xc1022407e0, 0xc008d100b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1022407b0, 0xc1028a8540) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10938779 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0faafa360) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10886711 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1b12b97c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0e24eb2c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1064c9380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10885805 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f578377c110, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc175f04018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc175f04000, 0xc237594000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc175f04000, 0xc237594000, 0x8000, 0x8000, 0x20302e, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc533eda020, 0xc237594000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc144658180, 0xc149b57458, 0x9, 0x9, 0xc000423880, 0x1007f57aae75f10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc144658180, 0xc149b57458, 0x9, 0x9, 0x9, 0x601742d5, 0xc1b8880dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc149b57458, 0x9, 0x9, 0x2b41140, 0xc144658180, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc149b57420, 0xc1b2054200, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc089678f00, 0xc15be761b0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc089678f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc089678f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10887530 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01ee85450, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc149d08420, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc143998900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942495 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1661e0300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0b2ec4570, 0xc1825b7a40, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0b2ec4570, 0xc1825b7a40, 0x25, 0x38d89c7c, 0x7c, 0x50a88eeb716338c4, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0b2ec4570, 0xc1825b7a40, 0x25, 0x38d89c7c, 0x7c, 0x2bd18c0, 0xc0caeb2fc0, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0cb47d580, 0xc1825b7a40, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x7000102, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0cb47d580, 0xc1825b7a40, 0x25, 0x38d89c7c, 0x7c, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0cb47d580, 0xc0640f9ec0, 0x20, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0cb47d580, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0b2ec4540, 0xc0cafe65a0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10902775 [semacquire]: | |
| sync.runtime_Semacquire(0xc0a040e9d0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0a040e9c8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0a040e9c0, 0xc0897a0ab0, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1b4617dc0, 0xc0fd8c83c0, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1882 +0x863 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1b4617dc0, 0xc16aa5bd00, 0x2, 0x2, 0xc183c56000, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0076a5680, 0xc1b4617dc0, 0xc14dd20900, 0xc16aa5bd00, 0x2, 0x2, 0xc101247c20, 0x2b44f80, 0xc0ac8ae900, 0x48, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x466644, 0xc2220dba00) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0ac8ae8d0, 0xc082f44ee0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10927216 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0b6ced6d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc185cec120, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc13c12d800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942791 [select]: | |
| net/http.(*persistConn).roundTrip(0xc111440240, 0xc1546c6840, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc003ace800, 0xc14e1fc6c0, 0xc030df2678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc003ace800, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc003ace800, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc54fd7a068, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc003ace800, 0x0, 0x0, 0x0, 0xc54fd7a068, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc003ace800, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc208a8a240, 0xc208a8a2a0, 0xc208a8a300, 0xc11b38e3f0, 0xc129c7e540, 0xc07ae6cf20, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4975789 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc07988d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10847635 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc125006d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10899543 [semacquire]: | |
| sync.runtime_Semacquire(0xc0b31d7e10) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0b31d7e08) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0b31d7e00, 0xc0dc560750, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0ef70a5c0, 0xc0fd8c8280, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1882 +0x863 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0ef70a5c0, 0xc0093c9810, 0x2, 0x2, 0xc16f7b8000, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0076a5680, 0xc0ef70a5c0, 0xc16f7b9380, 0xc0093c9810, 0x2, 0x2, 0xc16f7b86c0, 0x2b44f80, 0xc14c7a8180, 0x3, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x80, 0xc19ce14259) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc14c7a8150, 0xc03030e0e0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10846852 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f4ef52c40a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fb97d518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fb97d500, 0xc02f87a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fb97d500, 0xc02f87a000, 0x1000, 0x1000, 0x4db33a, 0xe7bef5f174ad, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc17a902818, 0xc02f87a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc05be541b0, 0xc02f87a000, 0x1000, 0x1000, 0xbffdeeaa3a945339, 0x381c61759d2, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc136e8ca20) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc136e8ca20, 0x4, 0x381c61759d2, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc15b7a6280, 0x2b99c80, 0xc14ae7c040) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 10026055 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc044d67200, 0xc0cd8d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc2295d9ce0, 0xc0cd8d6000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0ffb9b830, 0xc0cd8d6000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc101a76000, 0x7f56e2d6a100, 0xc0ffb9b830, 0xc0cd8d6000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc101a76000, 0x7f56e2d6a100, 0xc0ffb9b830, 0xc0cd8d6000, 0x8000, 0x8000, 0x2b6fc80, 0xc0ffb9b830, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0fb14bb80, 0x2b44ea0, 0xc101a76000, 0xc0cd8d6000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xd2ff8f94ae8c7601, 0x2de4cad2d1b0cb8f, 0xc156be14c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xd2ff8f94ae8c7601, 0x2de4cad2d1b0cb8f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xd2ff8f94ae8c7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0018d9d00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10938514 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f5c88900) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10026407 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc065ce6000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99c80, 0xc000fa0080, 0xc20cbb2ab0, 0x25, 0x18193527, 0x1bf5801, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99c80, 0xc000fa0080, 0xc20cbb2ab0, 0x25, 0x18193527, 0x1bf5801, 0xc08989d5c0, 0x40cfa8, 0x60, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99c80, 0xc000fa0080, 0xc20cbb2ab0, 0x25, 0x18193527, 0x1bf5801, 0x2bd1cc0, 0x3f98dc0, 0x2b99c80, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc000fa0080, 0xc20cbb2ab0, 0x25, 0x18193527, 0x1bf5801, 0x0, 0x5000104, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc000fa0080, 0xc20cbb2ab0, 0x25, 0x18193527, 0x1bf5801, 0x0, 0x7f56e2d6a100, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc000fa0080, 0xc1446a0b20, 0x20, 0x18193527, 0x1bf5801, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopyPostingsOffsets(0xc0f2165d00, 0x2b44ea0, 0xc1e6e00500, 0xc1ae5aa000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:218 +0xb7 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xb62701ba1657601, 0xde7b2b20a00aa6e3, 0xc16cc5a300, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:100 +0x389 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xb62701ba1657601, 0xde7b2b20a00aa6e3, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xb62701ba1657601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0035c4000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4864144 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc02ffa25f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc01ea19620, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1d99d5380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4976059 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc022eff7c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ac2a6fc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0f7624180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10851100 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0bf17a690, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc18add1aa0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc079683980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10902382 [semacquire]: | |
| sync.runtime_Semacquire(0xc23dae40d0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc23dae40c8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc23dae40c0, 0xc22be97290, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00bdb7f80, 0xc0fa553900, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1882 +0x863 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00bdb7f80, 0xc16aa5b270, 0x2, 0x2, 0xc0c8488330, 0xd, 0x1, 0xc0c57da700, 0x7f57aae7ba30) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0076a5680, 0xc00bdb7f80, 0xc101246ae0, 0xc16aa5b270, 0x2, 0x2, 0xc00bb03e00, 0x2b44f80, 0xc1a121a180, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1a121a150, 0xc08276a3f0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10928737 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc13c12d800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4964422 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1b4c86a50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1b364bbc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1b4efe780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10886261 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec70021f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1af554d18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1af554d00, 0xc162842000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1af554d00, 0xc162842000, 0x8000, 0x8000, 0x203025, 0x420bb5, 0xc094bb6020) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc122508358, 0xc162842000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1b64248a0, 0xc149f138b8, 0x9, 0x9, 0xc03053e380, 0x1007f57aae89f10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1b64248a0, 0xc149f138b8, 0x9, 0x9, 0x9, 0x601742e4, 0xc0020abdc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc149f138b8, 0x9, 0x9, 0x2b41140, 0xc1b64248a0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc149f13880, 0xc0a1432000, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1e04b4000, 0xc1b3427140, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1e04b4000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1e04b4000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942461 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0cc190120, 0xa, 0x2b99c80, 0xc0a94c7c80, 0xc0a040eb10) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 5013410 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1cb237980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4861133 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1b9453680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10026052 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1656e8300, 0xc1f1d06000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc0f06fc180, 0xc1f1d06000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0fc956120, 0xc1f1d06000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0e0b419a0, 0x7f56e2d6a100, 0xc0fc956120, 0xc1f1d06000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0e0b419a0, 0x7f56e2d6a100, 0xc0fc956120, 0xc1f1d06000, 0x8000, 0x8000, 0x2b6fc80, 0xc0fc956120, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1007aaf00, 0x2b44ea0, 0xc0e0b419a0, 0xc1f1d06000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x13bfff7f545b7601, 0x46f4c0748a74adcd, 0xc1668fe440, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x13bfff7f545b7601, 0x46f4c0748a74adcd, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x13bfff7f545b7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0006efa00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10885803 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc17d146fa0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1b776b3e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc089678f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10941455 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4ed3cf64c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0e83e5a98, 0x72, 0xc300, 0xc339, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0e83e5a80, 0xc1227a2000, 0xc339, 0xc339, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0e83e5a80, 0xc1227a2000, 0xc339, 0xc339, 0x203027, 0x752c50, 0xc0db9b32b8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc353b7cf10, 0xc1227a2000, 0xc339, 0xc339, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc09f99e1a0, 0xc1227a2000, 0xc339, 0xc339, 0x2588, 0x7ef2, 0xc1f1a60e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0db9b33d8, 0x2b41380, 0xc09f99e1a0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0db9b3180, 0x2b48520, 0xc353b7cf10, 0x5, 0xc353b7cf10, 0x2578) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0db9b3180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0db9b3180, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f31f7b00, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc18374daa0, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc189b81e00, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0fe116700, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0fe116700, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0fe116740, 0xc1f1f984aa, 0x3b56, 0x3b56, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0fe116740, 0xc1f1f94000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc024bb4420, 0xc024bb44e0, 0xc024bb4540, 0xc0f745a9a0, 0xc10317ad80, 0xc0f147e630, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4860163 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1efa63b80, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004963aa0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0aeaedb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4974784 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1e3012180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10885804 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc089678f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10026405 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc06743c300, 0xc1ff846000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc14087d500, 0xc1ff846000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0face7860, 0xc1ff846000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0db815630, 0x7f56e2d6a100, 0xc0face7860, 0xc1ff846000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0db815630, 0x7f56e2d6a100, 0xc0face7860, 0xc1ff846000, 0x8000, 0x8000, 0x2b6fc80, 0xc0face7860, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0ccfef200, 0x2b44ea0, 0xc0db815630, 0xc1ff846000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xe56ecbcc12c7701, 0x8b7d855f46ba2cb8, 0xc156be0e80, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xe56ecbcc12c7701, 0x8b7d855f46ba2cb8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xe56ecbcc12c7701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc008a67400, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4864309 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec5eca2a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0dccfc018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0dccfc000, 0xc13f02e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0dccfc000, 0xc13f02e000, 0x8000, 0x8000, 0x203011, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02ccd4c18, 0xc13f02e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1fd64a600, 0xc1271b30d8, 0x9, 0x9, 0xc0795a1880, 0x1007f57aae60080, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1fd64a600, 0xc1271b30d8, 0x9, 0x9, 0x9, 0x601742e4, 0xc0b5ff7dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1271b30d8, 0x9, 0x9, 0x2b41140, 0xc1fd64a600, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1271b30a0, 0xc085a9ff40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc180f4b080, 0xc0b9c9b3e0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc180f4b080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc180f4b080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 5005915 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc032a28300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10847634 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc06e9b4190, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc16500eba0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc125006d80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4964424 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f57840ad5f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1b2111e18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1b2111e00, 0xc1f9b42000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1b2111e00, 0xc1f9b42000, 0x8000, 0x8000, 0x203021, 0x420bb5, 0xc0868e4020) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc00da1a300, 0xc1f9b42000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1b2147260, 0xc05ddb9a78, 0x9, 0x9, 0xc000422380, 0x1007f57aae85180, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1b2147260, 0xc05ddb9a78, 0x9, 0x9, 0x9, 0x601742e4, 0xc06aed7dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc05ddb9a78, 0x9, 0x9, 0x2b41140, 0xc1b2147260, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc05ddb9a40, 0xc09403c000, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1b4efe780, 0xc1b4cee2d0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10886728 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4eca5ce930, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1af554d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1af554d80, 0xc1b85d2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1af554d80, 0xc1b85d2000, 0x8000, 0x8000, 0x203030, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc122508360, 0xc1b85d2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1b6425d40, 0xc0bb26b998, 0x9, 0x9, 0xc07aad6a80, 0x1007f57aae5de98, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1b6425d40, 0xc0bb26b998, 0x9, 0x9, 0x9, 0x601742da, 0xc0006d5dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0bb26b998, 0x9, 0x9, 0x2b41140, 0xc1b6425d40, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0bb26b960, 0xc0981a1720, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc163467380, 0xc198819d40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc163467380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc163467380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10938583 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56ba04cb28, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f9972718, 0x72, 0xf900, 0xf974, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f9972700, 0xc26bce6000, 0xf974, 0xf974, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f9972700, 0xc26bce6000, 0xf974, 0xf974, 0x203020, 0x53d49b, 0xc0f9b4e20c) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0e22f7c28, 0xc26bce6000, 0xf974, 0xf974, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc082803380, 0xc26bce6000, 0xf974, 0xf974, 0xc13e9ec908, 0x417a5e, 0xc13e9ec900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e505ccd8, 0x2b41380, 0xc082803380, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e505ca80, 0x2b48520, 0xc0e22f7c28, 0x5, 0xc0e22f7c28, 0x22e1fe0) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e505ca80, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e505ca80, 0xc0f9b4e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0df5ebb00, 0xc0f9b4e000, 0x1000, 0x1000, 0xc0aae2a360, 0xc13e9ecc58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc096d8cc60) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc096d8cc60, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc1a0f503c0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0df5ebb00) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4861134 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e151aa70, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc14554d098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc14554d080, 0xc09585a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc14554d080, 0xc09585a000, 0x8000, 0x8000, 0x203004, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc138d50688, 0xc09585a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc208a129c0, 0xc03b89c3b8, 0x9, 0x9, 0xc000581180, 0x1007f57aae847c0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc208a129c0, 0xc03b89c3b8, 0x9, 0x9, 0x9, 0x601742d5, 0xc074c04dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc03b89c3b8, 0x9, 0x9, 0x2b41140, 0xc208a129c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc03b89c380, 0xc0a156a880, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1b9453680, 0xc15ad7f740, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1b9453680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1b9453680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942783 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc0d7143500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc1afc79dd0, 0xc181ca3170, 0x25, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc1afc79dd0, 0xc181ca3170, 0x25, 0x327bf044, 0x64, 0x38360dd04b628542, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc1afc79dd0, 0xc181ca3170, 0x25, 0x327bf044, 0x64, 0x2bd18c0, 0xc0b005d0e0, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0a49e9640, 0xc181ca3170, 0x25, 0x327bf044, 0x64, 0x0, 0x4000105, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0a49e9640, 0xc181ca3170, 0x25, 0x327bf044, 0x64, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0a49e9640, 0xc1f6799c60, 0x20, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0a49e9640, 0x327bf044, 0x64, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0b50e71a0, 0xc09496cd80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942710 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4ec5eca7e8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1127b2398, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1127b2380, 0xc0ccf1e000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1127b2380, 0xc0ccf1e000, 0x109b, 0x109b, 0x203020, 0xc1127b2398, 0xc1127b2380) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02ffae320, 0xc0ccf1e000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc083acdb80, 0xc0ccf1e000, 0x109b, 0x109b, 0xcbb568f2e6ab6465, 0x59f111f13956c25b, 0xc076256900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e14ff758, 0x2b41380, 0xc083acdb80, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e14ff500, 0x2b48520, 0xc02ffae320, 0x5, 0xc02ffae320, 0xd5a79147c6e00bf3) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e14ff500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e14ff500, 0xc1759a4000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc111440240, 0xc1759a4000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc16bae1bc0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc16bae1bc0, 0x1, 0xc208a8aae0, 0x0, 0x91, 0xc02ddcd900, 0xc0be238580) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc111440240) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10942584 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc0d7143380, 0xc174b1c000, 0x3e80, 0x3e80, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc108fcc900, 0xc174b1c000, 0x3e80, 0x3e80, 0x101000000000000, 0x0, 0x3e80) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0b266d530, 0xc174b1c000, 0x3e80, 0x3e80, 0xc08edc3d50, 0x40a685, 0x21a56c0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.ReadAtLeast(0x7f56e2d6a100, 0xc0b266d530, 0xc174b1c000, 0x3e80, 0x3e80, 0x3e80, 0x3e80, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).fetchMissingSubranges.func1(0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:439 +0x407 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1afc79650, 0xc0cda27300) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10853742 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1629f0280, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1691ce0c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0de205200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4860375 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783047d28, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0795c5d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0795c5d80, 0xc08f12a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0795c5d80, 0xc08f12a000, 0x8000, 0x8000, 0x203022, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc00fa2af18, 0xc08f12a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc00bf47620, 0xc07d68cab8, 0x9, 0x9, 0xc00059ca80, 0x1007f57aae925d8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc00bf47620, 0xc07d68cab8, 0x9, 0x9, 0x9, 0x601742d5, 0xc171b1edc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc07d68cab8, 0x9, 0x9, 0x2b41140, 0xc00bf47620, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc07d68ca80, 0xc210097400, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1cf891b00, 0xc06e5cfe60, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1cf891b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1cf891b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942918 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4ee9e36088, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1124dc818, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1124dc800, 0xc0cf785300, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1124dc800, 0xc0cf785300, 0x109b, 0x109b, 0x20301c, 0xc02b463850, 0x417a5e) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02ffae358, 0xc0cf785300, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc0722527c0, 0xc0cf785300, 0x109b, 0x109b, 0x31621a06e5c40cdd, 0xc6677b7282f188fb, 0xc02b463900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc023124258, 0x2b41380, 0xc0722527c0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc023124000, 0x2b48520, 0xc02ffae358, 0x5, 0xc02ffae358, 0xc10145e3f6) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc023124000, 0x0, 0x0, 0x406) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc023124000, 0xc17692a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc20b290a20, 0xc17692a000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1ebd7ccc0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1ebd7ccc0, 0x1, 0xc20d237b00, 0xc0f9a5ae00, 0x412fae, 0xbebc200, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc20b290a20) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10942757 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0df5ebb00, 0xc0a9035b80, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0d6eb7a00, 0xc1d56ea120, 0xc0be36c678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0d6eb7a00, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0d6eb7a00, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc17bb78150, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0d6eb7a00, 0x0, 0x0, 0x0, 0xc17bb78150, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0d6eb7a00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc215d62d20, 0xc215d62d80, 0xc215d62de0, 0xc0bae6be40, 0xc11a7d6180, 0xc0bf8434a0, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10886727 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc163467380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10941467 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc0831d9d40, 0xa, 0x2b99c80, 0xc1ed2d2180, 0xc23dae4210) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10942983 [select]: | |
| net/http.(*persistConn).writeLoop(0xc10c16ac60) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10896901 [semacquire]: | |
| sync.runtime_Semacquire(0xc16532cb80) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc16532cb78) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc16532cb70, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1841ced20) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc16532c9f0, 0x255e040, 0x18, 0xc032c873f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1b6870900, 0x2bbccc0, 0xc046582540, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1b6870900, 0x7f57841eead0, 0xc16ff49a00, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1b6870900, 0x2bbbd60, 0xc16ff49a00, 0xc00083ae00, 0xc089c79688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc0465824a0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc046582480, 0xc046582320, 0x26b3908, 0xc046582480, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc046582480, 0x2b99d40, 0xc16532c870) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc046582460, 0xc046582320, 0xc046582340, 0x7f57aae60080, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc046582460, 0xc046582320, 0xc046582340, 0x2b99d01, 0xc046582460) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc046582460, 0x2b99d40, 0xc16532c840) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0xc046582320, 0xc046582360, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0x1, 0xc0465822e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0xc046582320, 0xc046582380, 0x2a0be560, 0xe7bee5d58cb3) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0x203011, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0xc046582320, 0xc0465823a0, 0xc046582320, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0xc0017b8c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3ec00, 0xc046582320, 0x26b3908, 0x2b99d40, 0xc16532c7e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc033289f00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc033289f00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc09ce7f0e0, 0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc033289f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10026401 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc16ba3f200, 0xc1a387c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc18374db00, 0xc1a387c000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc100c5ab40, 0xc1a387c000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0f25eaff0, 0x7f56e2d6a100, 0xc100c5ab40, 0xc1a387c000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0f25eaff0, 0x7f56e2d6a100, 0xc100c5ab40, 0xc1a387c000, 0x8000, 0x8000, 0x2b6fc80, 0xc100c5ab40, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc088033280, 0x2b44ea0, 0xc0f25eaff0, 0xc1a387c000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x581bc9f30e667601, 0x145ef23d54455c8f, 0xc15b5a05c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x581bc9f30e667601, 0x145ef23d54455c8f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x581bc9f30e667601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc000fa7400, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10898361 [semacquire]: | |
| sync.runtime_Semacquire(0xc149156f70) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc149156f68) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc149156f60, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc153c4d1d0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc149156db0, 0x255e040, 0x18, 0xc0ae64f3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc0dd83a2a0, 0x2bbccc0, 0xc212562b60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc0dd83a2a0, 0x7f57841eead0, 0xc000404950, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc0dd83a2a0, 0x2bbbd60, 0xc000404950, 0xc00083ae00, 0xc08e51d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc212562ac0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc212562aa0, 0xc212562920, 0x26b3908, 0xc212562aa0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc212562aa0, 0x2b99d40, 0xc149156c60) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc212562a80, 0xc212562920, 0xc212562940, 0x7f57aae87a30, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc212562a80, 0xc212562920, 0xc212562940, 0x2b99d01, 0xc212562a80) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc212562a80, 0x2b99d40, 0xc149156c30) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0xc212562920, 0xc212562960, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0x1, 0xc2125628e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0xc212562920, 0xc212562980, 0x1340a4c2, 0xe7becf0a4c3b) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0x203084, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0xc212562920, 0xc2125629a0, 0xc212562920, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0xc4cfcf7c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89c80, 0xc212562920, 0x26b3908, 0x2b99d40, 0xc149156bd0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc125006d80, 0xc06a2b4b00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc125006d80, 0xc06a2b4b00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc108876de0, 0xc000c0cea0, 0x2bc6d40, 0xc125006d80, 0xc06a2b4b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10902956 [semacquire]: | |
| sync.runtime_Semacquire(0xc0b2ec4550) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0b2ec4548) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0b2ec4540, 0xc0cafe65a0, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00668e180, 0xc0fd8c8500, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1882 +0x863 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00668e180, 0xc06f4a9960, 0x2, 0x2, 0xc1117d36e0, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0076a5680, 0xc00668e180, 0xc11efdf080, 0xc06f4a9960, 0x2, 0x2, 0xc11efde360, 0x2b44f80, 0xc089aa9860, 0x61507a7061776e42, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x572f77776d41746c, 0x674c346e6d664331) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc089aa9830, 0xc0830f09a0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10886726 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc185ec6f50, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1a31ed9e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc163467380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10886259 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1e6ba0190, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc18cf99680, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1e04b4000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10938586 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f8ec2120) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10026060 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc16687f080, 0xc1f1d42000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc200eb68a0, 0xc1f1d42000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0fb5e2420, 0xc1f1d42000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0dd17ca50, 0x7f56e2d6a100, 0xc0fb5e2420, 0xc1f1d42000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0dd17ca50, 0x7f56e2d6a100, 0xc0fb5e2420, 0xc1f1d42000, 0x8000, 0x8000, 0x2b6fc80, 0xc0fb5e2420, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0f2165f00, 0x2b44ea0, 0xc0dd17ca50, 0xc1f1d42000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x9b590d27b8a77601, 0x9c2a4a2cac279d9a, 0xc15b5a15c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x9b590d27b8a77601, 0x9c2a4a2cac279d9a, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x9b590d27b8a77601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0049db600, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4860373 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1efe73b30, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0d4d33260, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1cf891b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4860026 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0ba5f7c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10928738 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4edb485fd0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1d43e9b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1d43e9b00, 0xc0d8b98000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1d43e9b00, 0xc0d8b98000, 0x8000, 0x8000, 0x20302e, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1d58829b8, 0xc0d8b98000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1422103c0, 0xc12ff56b98, 0x9, 0x9, 0xc002e00380, 0x1007f57aae75180, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1422103c0, 0xc12ff56b98, 0x9, 0x9, 0x9, 0x601742d1, 0xc196ffbdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc12ff56b98, 0x9, 0x9, 0x2b41140, 0xc1422103c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc12ff56b60, 0xc2103d8e20, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc13c12d800, 0xc0d14aa2a0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc13c12d800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc13c12d800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10897918 [semacquire]: | |
| sync.runtime_Semacquire(0xc1af1b9900) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1af1b98f8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1af1b98f0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0ef228360) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1af1b91d0, 0x255e040, 0x18, 0xc0cdfe33f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc146ff9f20, 0x2bbccc0, 0xc22adffc20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc146ff9f20, 0x7f57841eead0, 0xc00929d380, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc146ff9f20, 0x2bbbd60, 0xc00929d380, 0xc00083ae00, 0xc00d45d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc22adffb80, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc22adffb60, 0xc22adffa00, 0x26b3908, 0xc22adffb60, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc22adffb60, 0x2b99d40, 0xc1af1b9050) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc22adffb40, 0xc22adffa00, 0xc22adffa20, 0x7f57aae6eca0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc22adffb40, 0xc22adffa00, 0xc22adffa20, 0x2b99d01, 0xc22adffb40) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc22adffb40, 0x2b99d40, 0xc1af1b9020) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0xc22adffa00, 0xc22adffa40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0x1, 0xc22adff9c0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0xc22adffa00, 0xc22adffa60, 0x189c78a8, 0xe7be98cb5600) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0x20308a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0xc22adffa00, 0xc22adffa80, 0xc22adffa00, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0xc0815e8c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23c88f500, 0xc22adffa00, 0x26b3908, 0x2b99d40, 0xc1af1b8fc0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc133bf6300, 0xc06f0eaa00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc133bf6300, 0xc06f0eaa00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc009006804, 0xc000c0cea0, 0x2bc6d40, 0xc133bf6300, 0xc06f0eaa00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10942560 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0caeb3200) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10900514 [semacquire]: | |
| sync.runtime_Semacquire(0xc1a121a160) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1a121a158) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1a121a150, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc153a0d6e0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc19d1f2db0, 0x255e040, 0x18, 0xc0b02c53f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc00bb03e00, 0x2bbccc0, 0xc027ee76c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc00bb03e00, 0x7f57841eead0, 0xc16aa5a810, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc00bb03e00, 0x2bbbd60, 0xc16aa5a810, 0xc00083ae00, 0xc0317af688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc027ee7620, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc027ee7600, 0xc027ee74a0, 0x26b3908, 0xc027ee7600, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc027ee7600, 0x2b99d40, 0xc19d1f2c00) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc027ee75e0, 0xc027ee74a0, 0xc027ee74c0, 0x7f57aae8b368, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc027ee75e0, 0xc027ee74a0, 0xc027ee74c0, 0x2b99d01, 0xc027ee75e0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc027ee75e0, 0x2b99d40, 0xc19d1f2bd0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0xc027ee74a0, 0xc027ee74e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0x1, 0xc027ee7460) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0xc027ee74a0, 0xc027ee7500, 0x2a112b16, 0xe7bee5dad273) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0x203009, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0xc027ee74a0, 0xc027ee7520, 0xc027ee74a0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0xc10e4c0c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c0480, 0xc027ee74a0, 0x26b3908, 0x2b99d40, 0xc19d1f2b70) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780, 0xc10a00c800, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780, 0xc10a00c800, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc18c975ff0, 0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780, 0xc10a00c800) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10844990 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01b3e0aa0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc23b758ae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc07ab0a600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10844991 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc07ab0a600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942590 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc103ab9aa0, 0xa, 0x2b99c80, 0xc0a9ee7800, 0xc1afc79ec0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10929539 [select]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1a0b78000, 0xc1a0b780c0, 0xc1a0b78120, 0xc1010eadc0, 0xc0d6946120, 0xc076cb5d90, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10899183 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc09c457fe0, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00236c700, 0xc0cc896180, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00236c700, 0xc002ebdd80, 0x2, 0x2, 0xc11f72a780, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc00236c700, 0xc163049aa0, 0xc002ebdd80, 0x2, 0x2, 0xc14642de60, 0x2b44f80, 0xc15da39a40, 0xc0005d4428, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc15da39a10, 0xc0f3fb99d0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4864210 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783d0d128, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc11f665798, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc11f665780, 0xc13e96c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc11f665780, 0xc13e96c000, 0x8000, 0x8000, 0x20300d, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1040ccf60, 0xc13e96c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0076a6b40, 0xc0c068a578, 0x9, 0x9, 0xc000423c00, 0x1007f57aae707c0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0076a6b40, 0xc0c068a578, 0x9, 0x9, 0x9, 0x601742e4, 0xc01c35adc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0c068a578, 0x9, 0x9, 0x2b41140, 0xc0076a6b40, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c068a540, 0xc0f5d41560, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1ce23d680, 0xc0334086f0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1ce23d680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1ce23d680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10938383 [select]: | |
| net/http.(*persistConn).readLoop(0xc0d965ac60) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10940897 [semacquire]: | |
| sync.runtime_Semacquire(0xc101475d50) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc101475d48) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc101475d40, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc10278b5f0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc101475bc0, 0x255e040, 0x18, 0xc12e3b93f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc13531f260, 0x2bbccc0, 0xc1ba6dd200, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc13531f260, 0x7f57841eead0, 0xc101223eb0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc13531f260, 0x2bbbd60, 0xc101223eb0, 0xc00083ae00, 0xc11b11d688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc1ba6dd160, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc1ba6dd140, 0xc1ba6dcfe0, 0x26b3908, 0xc1ba6dd140, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc1ba6dd140, 0x2b99d40, 0xc101475a70) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc1ba6dd120, 0xc1ba6dcfe0, 0xc1ba6dd000, 0x7f57aae87368, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc1ba6dd120, 0xc1ba6dcfe0, 0xc1ba6dd000, 0x2b99d01, 0xc1ba6dd120) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc1ba6dd120, 0x2b99d40, 0xc101475a40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0xc1ba6dcfe0, 0xc1ba6dd020, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0x1, 0xc1ba6dcfa0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0xc1ba6dcfe0, 0xc1ba6dd040, 0x2bd82fe1, 0xe7c38fb99f37) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0x20306e, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x7f4ec41cc000, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0xc1ba6dcfe0, 0xc1ba6dd060, 0xc1ba6dcfe0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0xc001c89c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc19a757380, 0xc1ba6dcfe0, 0x26b3908, 0x2b99d40, 0xc1014759e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc101456c00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc101456c00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc16ac5fa50, 0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc101456c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10850786 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0a5e3c780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10900515 [semacquire]: | |
| sync.runtime_Semacquire(0xc0ac8ae8e0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0ac8ae8d8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0ac8ae8d0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc2184186c0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc153a0df80, 0x255e040, 0x18, 0xc0be3713f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc101247c20, 0x2bbccc0, 0xc129aa6b60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc101247c20, 0x7f57841eead0, 0xc16aa5b2d0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc101247c20, 0x2bbbd60, 0xc16aa5b2d0, 0xc00083ae00, 0xc0317b3688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc129aa6aa0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc129aa6a80, 0xc129aa68e0, 0x26b3908, 0xc129aa6a80, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc129aa6a80, 0x2b99d40, 0xc153a0d8c0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc129aa6a60, 0xc129aa68e0, 0xc129aa6900, 0x7f57aae8b368, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc129aa6a60, 0xc129aa68e0, 0xc129aa6900, 0x2b99d01, 0xc129aa6a60) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc129aa6a60, 0x2b99d40, 0xc153a0d890) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0xc129aa68e0, 0xc129aa6920, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0x1, 0xc129aa68a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0xc129aa68e0, 0xc129aa6940, 0x2a29ce74, 0xe7bee5f375d0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0x20304a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0xc129aa68e0, 0xc129aa6960, 0xc129aa68e0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0xc10e4c0c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0933c1200, 0xc129aa68e0, 0x26b3908, 0x2b99d40, 0xc153a0d7a0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780, 0xc10a00c900, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780, 0xc10a00c900, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc18c975ff0, 0xc000c0cea0, 0x2bc6d40, 0xc1b4efe780, 0xc10a00c900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10938384 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0d965ac60) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10942919 [select]: | |
| net/http.(*persistConn).writeLoop(0xc20b290a20) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10884840 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1b1ab0190, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc144952540, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc028320300) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10943012 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f9db57a0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4859011 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15db39270, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc02d48dbc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1b9453380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10895438 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00c07ff90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc148cea2a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc165941200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10026063 [runnable]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc03f222a80, 0xc1bdd80000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc18374db60, 0xc1bdd80000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc100c5ac60, 0xc1bdd80000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0e3071040, 0x7f56e2d6a100, 0xc100c5ac60, 0xc1bdd80000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0e3071040, 0x7f56e2d6a100, 0xc100c5ac60, 0xc1bdd80000, 0x8000, 0x8000, 0x2b6fc80, 0xc100c5ac60, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1007aa500, 0x2b44ea0, 0xc0e3071040, 0xc1bdd80000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x1d98e6dc8e037701, 0x737752b7f95d9f59, 0xc1a0bb6700, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x1d98e6dc8e037701, 0x737752b7f95d9f59, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x1d98e6dc8e037701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc004be0700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10893275 [semacquire]: | |
| sync.runtime_Semacquire(0xc02b22f4e0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc02b22f4d8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc02b22f4d0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc1534aeff0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc02b22efc0, 0x255e040, 0x18, 0xc2107bf3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc11d7a2540, 0x2bbccc0, 0xc12a826f40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc11d7a2540, 0x7f57841eead0, 0xc16cee7920, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc11d7a2540, 0x2bbbd60, 0xc16cee7920, 0xc00083ae00, 0xc0313bd688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc12a826ea0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc12a826e80, 0xc12a826d20, 0x26b3908, 0xc12a826e80, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc12a826e80, 0x2b99d40, 0xc02b22ee70) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc12a826e60, 0xc12a826d20, 0xc12a826d40, 0x7f57aae71848, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc12a826e60, 0xc12a826d20, 0xc12a826d40, 0x2b99d01, 0xc12a826e60) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc12a826e60, 0x2b99d40, 0xc02b22ee40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0xc12a826d20, 0xc12a826d60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0x1, 0xc12a826ce0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0xc12a826d20, 0xc12a826d80, 0x2a1e99e6, 0xe7bee5e8413e) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0x20304a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0xc12a826d20, 0xc12a826da0, 0xc12a826d20, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0xc12c1aec68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc11a382b40, 0xc12a826d20, 0x26b3908, 0x2b99d40, 0xc02b22ede0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780, 0xc136aba100, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780, 0xc136aba100, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc07f479550, 0xc000c0cea0, 0x2bc6d40, 0xc0a5e3c780, 0xc136aba100) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4863968 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc258a514f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0076a6de0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1ce23d680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10844992 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed2d21828, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0ddf2a398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0ddf2a380, 0xc0b0390000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0ddf2a380, 0xc0b0390000, 0x8000, 0x8000, 0x20305b, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1c7b3e358, 0xc0b0390000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc168230a20, 0xc07ba8fd18, 0x9, 0x9, 0xc0dd272e00, 0x1007f57aae6df10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc168230a20, 0xc07ba8fd18, 0x9, 0x9, 0x9, 0x601742e4, 0xc14cf35dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc07ba8fd18, 0x9, 0x9, 0x2b41140, 0xc168230a20, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc07ba8fce0, 0xc0a1432420, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc07ab0a600, 0xc15d3f4ab0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc07ab0a600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc07ab0a600) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10938509 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0e809d8c0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4861510 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec2adc950, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0a8708b18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0a8708b00, 0xc0911aa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0a8708b00, 0xc0911aa000, 0x8000, 0x8000, 0x203021, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc030498358, 0xc0911aa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc25236dbc0, 0xc072e4eab8, 0x9, 0x9, 0xc002e01180, 0x1007f57aae807c0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc25236dbc0, 0xc072e4eab8, 0x9, 0x9, 0x9, 0x601742d0, 0xc1f571fdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc072e4eab8, 0x9, 0x9, 0x2b41140, 0xc25236dbc0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc072e4ea80, 0xc214fe9fa0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc06b2d7500, 0xc06a733860, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc06b2d7500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc06b2d7500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942643 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc228bed800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0a7b74b40, 0xc20db619b0, 0x25, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0a7b74b40, 0xc20db619b0, 0x25, 0x327bf044, 0x64, 0x2eeb97cae2a265ff, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0a7b74b40, 0xc20db619b0, 0x25, 0x327bf044, 0x64, 0x2bd18c0, 0xc0adf4efc0, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0f83a0800, 0xc20db619b0, 0x25, 0x327bf044, 0x64, 0x0, 0x4000105, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0f83a0800, 0xc20db619b0, 0x25, 0x327bf044, 0x64, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0f83a0800, 0xc20f45bc20, 0x20, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0f83a0800, 0x327bf044, 0x64, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0xc0b60359e0, 0xc0fa2782f0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc02b217ef0, 0xc0bcb56360) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 5072029 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed32a4f08, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc21bd17d18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc21bd17d00, 0xc21c634000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc21bd17d00, 0xc21c634000, 0x8000, 0x8000, 0x20302a, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1bf002ad8, 0xc21c634000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc21c630240, 0xc06f433618, 0x9, 0x9, 0xc0b02e5c00, 0x1007f57aae60748, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc21c630240, 0xc06f433618, 0x9, 0x9, 0x9, 0x601742e2, 0xc001d07dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc06f433618, 0x9, 0x9, 0x2b41140, 0xc21c630240, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc06f4335e0, 0xc1ee578c60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc06362ed80, 0xc21bd7f590, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc06362ed80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc06362ed80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10897060 [semacquire]: | |
| sync.runtime_Semacquire(0xc14f9bda80) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc14f9bda78) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc14f9bda70, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc11db30030) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc14f9bd8f0, 0x255e040, 0x18, 0xc0b9f513f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc009e8bb60, 0x2bbccc0, 0xc02fae8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc009e8bb60, 0x7f57841eead0, 0xc009460370, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc009e8bb60, 0x2bbbd60, 0xc009460370, 0xc00083ae00, 0xc00acdd688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc14a1c5f60, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc14a1c5f40, 0xc14a1c5de0, 0x26b3908, 0xc14a1c5f40, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc14a1c5f40, 0x2b99d40, 0xc14f9bd740) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc14a1c5f20, 0xc14a1c5de0, 0xc14a1c5e00, 0x7f57aae77a30, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc14a1c5f20, 0xc14a1c5de0, 0xc14a1c5e00, 0x2b99d01, 0xc14a1c5f20) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc14a1c5f20, 0x2b99d40, 0xc14f9bd710) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0xc14a1c5de0, 0xc14a1c5e20, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0x1, 0xc14a1c5da0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0xc14a1c5de0, 0xc14a1c5e40, 0x2a130b53, 0xe7bee5dcb2a8) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0x203052, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0xc14a1c5de0, 0xc14a1c5e60, 0xc14a1c5de0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0xc0ad869c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc0dd654a80, 0xc14a1c5de0, 0x26b3908, 0x2b99d40, 0xc14f9bd680) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc03f990400, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc03f990400, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc16ac5fa50, 0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00, 0xc03f990400) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10026403 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc166396f00, 0xc1b0f86000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc10cdc47e0, 0xc1b0f86000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc1038228d0, 0xc1b0f86000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0db44bea0, 0x7f56e2d6a100, 0xc1038228d0, 0xc1b0f86000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0db44bea0, 0x7f56e2d6a100, 0xc1038228d0, 0xc1b0f86000, 0x8000, 0x8000, 0x2b6fc80, 0xc1038228d0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc103742780, 0x2b44ea0, 0xc0db44bea0, 0xc1b0f86000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xcf000366c9417701, 0x9e79dbe86788e879, 0xc11b7d1f80, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xcf000366c9417701, 0x9e79dbe86788e879, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xcf000366c9417701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc009211100, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10900993 [semacquire]: | |
| sync.runtime_Semacquire(0xc1984dae20) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc1984dae18) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc1984dae10, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc12f674f90) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc1984dac90, 0x255e040, 0x18, 0xc03456f3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1d56ebc80, 0x2bbccc0, 0xc02d44eac0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1d56ebc80, 0x7f57841eead0, 0xc00be2dae0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1d56ebc80, 0x2bbbd60, 0xc00be2dae0, 0xc00083ae00, 0xc04407b688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc02d44ea20, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc02d44ea00, 0xc02d44e8a0, 0x26b3908, 0xc02d44ea00, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc02d44ea00, 0x2b99d40, 0xc144a97ef0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02d44e9e0, 0xc02d44e8a0, 0xc02d44e8c0, 0x7f57aae76ca0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02d44e9e0, 0xc02d44e8a0, 0xc02d44e8c0, 0x2b99d01, 0xc02d44e9e0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc02d44e9e0, 0x2b99d40, 0xc144a97ec0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0xc02d44e8a0, 0xc02d44e8e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0x1, 0xc02d44e860) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0xc02d44e8a0, 0xc02d44e900, 0x2a152adf, 0xe7bee5ded23a) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0x20300b, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0xc02d44e8a0, 0xc02d44e920, 0xc02d44e8a0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0xc01a45fc68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc108b6ab40, 0xc02d44e8a0, 0x26b3908, 0x2b99d40, 0xc144a97e60) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc1dff65380, 0xc079e2cd00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc1dff65380, 0xc079e2cd00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc19ee657d0, 0xc000c0cea0, 0x2bc6d40, 0xc1dff65380, 0xc079e2cd00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10942496 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0caeb3200, 0xc0cb47d700, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0c77f3400, 0xc18c0cbce0, 0xc01a0f8678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0c77f3400, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0c77f3400, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc242704120, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0c77f3400, 0x0, 0x0, 0x0, 0xc242704120, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0c77f3400, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc205de55c0, 0xc205de5620, 0xc205de5680, 0xc0c3ed80e0, 0xc0b2ec45d0, 0xc0d13b4a50, 0x2b99d40, 0xc0b2ec4570, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10026056 [runnable]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1661d2f00, 0xc1a42f2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc200eb6480, 0xc1a42f2000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0e27c2c60, 0xc1a42f2000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0fcfbe820, 0x7f56e2d6a100, 0xc0e27c2c60, 0xc1a42f2000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0fcfbe820, 0x7f56e2d6a100, 0xc0e27c2c60, 0xc1a42f2000, 0x8000, 0x8000, 0x2b6fc80, 0xc0e27c2c60, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0f2165900, 0x2b44ea0, 0xc0fcfbe820, 0xc1a42f2000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x72f0a5b33be77601, 0xe929a56b0e3d46a6, 0xc15b5a1bc0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x72f0a5b33be77601, 0xe929a56b0e3d46a6, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x72f0a5b33be77601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc002692700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10942982 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4effd941e0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc17a6ae598, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc17a6ae580, 0xc228fecc00, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc17a6ae580, 0xc228fecc00, 0x109b, 0x109b, 0x20301c, 0xc0fad73218, 0xc0fad73200) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc54fd7a0b8, 0xc228fecc00, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc072250240, 0xc228fecc00, 0x109b, 0x109b, 0xcb25fe82412abc83, 0x59f111f13956c25b, 0xc190603900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc121a1f058, 0x2b41380, 0xc072250240, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc121a1ee00, 0x2b48520, 0xc54fd7a0b8, 0x5, 0xc54fd7a0b8, 0xd5a79147c6e00bf3) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc121a1ee00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc121a1ee00, 0xc1693ec000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc10c16ac60, 0xc1693ec000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc2a0a054a0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc2a0a054a0, 0x1, 0xc2072b18c0, 0xc1880bfc00, 0xda, 0xc03778a460, 0xc0b75d7580) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc10c16ac60) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10902076 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0xc1fb95e100, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc135d18880, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc1351a08c0, 0xc1065e8300, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc1351a08c0, 0xc06f4a8e40, 0x2, 0x2, 0xc1117d36e0, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc1351a08c0, 0xc30861e9c0, 0xc06f4a8e40, 0x2, 0x2, 0xc30861e600, 0x2b44f80, 0xc00c808090, 0x706f74050201c6e7, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x2e63696c6275702e, 0x30325f656c616373) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc00c808060, 0xc082c1d650) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10853744 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed0bf8748, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000f63d98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000f63d80, 0xc105b24000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc000f63d80, 0xc105b24000, 0x8000, 0x8000, 0x20306f, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1954441b0, 0xc105b24000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc13f94e0c0, 0xc1493e2578, 0x9, 0x9, 0xc002e01180, 0x1007f57aae5ec28, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc13f94e0c0, 0xc1493e2578, 0x9, 0x9, 0x9, 0x601742e4, 0xc126541dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1493e2578, 0x9, 0x9, 0x2b41140, 0xc13f94e0c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1493e2540, 0xc0f525d500, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0de205200, 0xc1760e8c90, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0de205200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0de205200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10026054 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1ed120d80, 0xc1ab8e6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc1d4adfaa0, 0xc1ab8e6000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0d69464e0, 0xc1ab8e6000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0f825fe50, 0x7f56e2d6a100, 0xc0d69464e0, 0xc1ab8e6000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0f825fe50, 0x7f56e2d6a100, 0xc0d69464e0, 0xc1ab8e6000, 0x8000, 0x8000, 0x2b6fc80, 0xc0d69464e0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc088033080, 0x2b44ea0, 0xc0f825fe50, 0xc1ab8e6000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xa2a6b19814497701, 0x20594ccb89f1a324, 0xc12125c940, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xa2a6b19814497701, 0x20594ccb89f1a324, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xa2a6b19814497701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc009b84700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 4863111 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1d99d4c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4864307 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc096a4aa00, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1fd64a660, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc180f4b080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942751 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0adf4f320, 0xc0a92214c0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0a53d2f00, 0xc168c96b40, 0xc107f66678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0a53d2f00, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0a53d2f00, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc029716100, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0a53d2f00, 0x0, 0x0, 0x0, 0xc029716100, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0a53d2f00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc20ce26360, 0xc20ce26420, 0xc20ce26480, 0xc163010870, 0xc0a7b74ba0, 0xc0c9342f20, 0x2b99d40, 0xc0a7b74b40, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10941466 [select]: | |
| net/http.(*persistConn).roundTrip(0xc02346d0e0, 0xc1ed2d2280, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc006140f00, 0xc005030060, 0xc03456a678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc006140f00, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc006140f00, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc122508020, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc006140f00, 0x0, 0x0, 0x0, 0xc122508020, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc006140f00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0831d9b60, 0xc0831d9bc0, 0xc0831d9c80, 0xc10217c1d0, 0xc23dae4150, 0xc0f79fc210, 0x2b99d40, 0xc23dae40f0, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10939867 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec2adc338, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f869f918, 0x72, 0xe400, 0xe4a1, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f869f900, 0xc26bc9a000, 0xe4a1, 0xe4a1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f869f900, 0xc26bc9a000, 0xe4a1, 0xe4a1, 0x203020, 0x752c50, 0xc06c1ba838) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0bae50438, 0xc26bc9a000, 0xe4a1, 0xe4a1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc080f019c0, 0xc26bc9a000, 0xe4a1, 0xe4a1, 0x2988, 0x8132, 0xc0d227ee90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc06c1ba958, 0x2b41380, 0xc080f019c0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc06c1ba700, 0x2b48520, 0xc0bae50438, 0x5, 0xc0bae50438, 0x2978) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc06c1ba700, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc06c1ba700, 0xc1a387f37a, 0x4c86, 0x4c86, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f5c88b40, 0xc1a387f37a, 0x4c86, 0x4c86, 0x2978, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc196cb6a80, 0xc1a387f37a, 0x4c86, 0x4c86, 0x2978, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc0a5afb140, 0xc1a387f37a, 0x4c86, 0x4c86, 0x2978, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0ffe9e280, 0xc1a387f37a, 0x4c86, 0x4c86, 0x2978, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0ffe9e280, 0xc1a387f37a, 0x4c86, 0x4c86, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0ffe9e2c0, 0xc1a387f37a, 0x4c86, 0x4c86, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0ffe9e2c0, 0xc1a387c000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc01ada60c0, 0xc01ada61e0, 0xc01ada6360, 0xc0f2d58940, 0xc100192090, 0xc0dde02b00, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10938048 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f5c88b40) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10942559 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4eca5cee70, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00b693018, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00b693000, 0xc0335c6000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00b693000, 0xc0335c6000, 0x109b, 0x109b, 0x20304d, 0xc0cfabba98, 0xc0cfabba80) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1538a8130, 0xc0335c6000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc137034460, 0xc0335c6000, 0x109b, 0x109b, 0xcba5f5ac3919012c, 0x59f111f13956c25b, 0xc196ffe900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e55cc5d8, 0x2b41380, 0xc137034460, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e55cc380, 0x2b48520, 0xc1538a8130, 0x5, 0xc1538a8130, 0xd5a79147c6e00bf3) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e55cc380, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e55cc380, 0xc176098000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0caeb3200, 0xc176098000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1a1673680) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1a1673680, 0x1, 0xc208a8b3e0, 0xc0e55cc380, 0x91, 0xc1837ae3c0, 0xc0cc332d80) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0caeb3200) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10939647 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed0bf7b28, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0e8da6f18, 0x72, 0xdf00, 0xdf14, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0e8da6f00, 0xc26bc7e000, 0xdf14, 0xdf14, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0e8da6f00, 0xc26bc7e000, 0xdf14, 0xdf14, 0x20302b, 0x752c50, 0xc0e55cd2b8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1d5882b80, 0xc26bc7e000, 0xdf14, 0xdf14, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc0ad134f80, 0xc26bc7e000, 0xdf14, 0xdf14, 0x2588, 0x30fe, 0xc0b02c8e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e55cd3d8, 0x2b41380, 0xc0ad134f80, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e55cd180, 0x2b48520, 0xc1d5882b80, 0x5, 0xc1d5882b80, 0x2578) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e55cd180, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e55cd180, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0d872d560, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc08e4e8d80, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc0cd9e0540, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0ff696a40, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x2578, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0ff696a40, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0ff696a80, 0xc1ab8e9d1a, 0x42e6, 0x42e6, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0ff696a80, 0xc1ab8e6000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1dbaf4840, 0xc1dbaf48a0, 0xc1dbaf4a80, 0xc0d5a01e10, 0xc0a490fdd0, 0xc0fa744630, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10895788 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc157d7d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942800 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc208a8acc0, 0xa, 0x2b99c80, 0xc1546c69c0, 0xc129c7ec30) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10942886 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4ef59619c8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1127b2798, 0x72, 0x1000, 0x10fb, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1127b2780, 0xc078016000, 0x10fb, 0x10fb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1127b2780, 0xc078016000, 0x10fb, 0x10fb, 0x203045, 0xc0f6bd50e0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0dd632038, 0xc078016000, 0x10fb, 0x10fb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc115c5a3e0, 0xc078016000, 0x10fb, 0x10fb, 0x1, 0x1, 0xc067330900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0c92ea5d8, 0x2b41380, 0xc115c5a3e0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0c92ea380, 0x2b48520, 0xc0dd632038, 0x5, 0xc0dd632038, 0x4daf20) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0c92ea380, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0c92ea380, 0xc176f8e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0adf4f320, 0xc176f8e000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc17b484b40) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc17b484b40, 0x1, 0xc20a7f4ba0, 0xc168a21cb8, 0x650bd3, 0xc149156db0, 0x216f320) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0adf4f320) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10938548 [select]: | |
| net/http.(*persistConn).readLoop(0xc0d872d560) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 5015585 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1cbf0fe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10940420 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f67b1440) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10942459 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc246e44d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0a040e9f0, 0xc1f6630f90, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0a040e9f0, 0xc1f6630f90, 0x25, 0x38d89c7c, 0x7c, 0x6577936a20529752, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0a040e9f0, 0xc1f6630f90, 0x25, 0x38d89c7c, 0x7c, 0x2bd18c0, 0xc0ae1d9440, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0a94c7c00, 0xc1f6630f90, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x4000105, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0a94c7c00, 0xc1f6630f90, 0x25, 0x38d89c7c, 0x7c, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0a94c7c00, 0xc206e66760, 0x20, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0a94c7c00, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0xc0e14ff500, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0a040e9c0, 0xc0897a0ab0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942913 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc205de5740, 0xa, 0x2b99c80, 0xc0cb47d600, 0xc0b2ec4660) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4860165 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783d33200, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc19062b698, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc19062b680, 0xc0894aa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc19062b680, 0xc0894aa000, 0x8000, 0x8000, 0x203024, 0x0, 0x60125575cc0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc18d208598, 0xc0894aa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1ed49fe00, 0xc13bf79538, 0x9, 0x9, 0xc0afff0e00, 0x1007f57aae76ca0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1ed49fe00, 0xc13bf79538, 0x9, 0x9, 0x9, 0x601742d8, 0xc125575dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc13bf79538, 0x9, 0x9, 0x2b41140, 0xc1ed49fe00, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc13bf79500, 0xc1fab7c800, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0aeaedb00, 0xc1cc4fbb00, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0aeaedb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0aeaedb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10943010 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0ae1d9680) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10901537 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc198b45480, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0a09a6cc0, 0xc022ed9500, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0a09a6cc0, 0xc16ff49950, 0x2, 0x2, 0xc1385a1c60, 0xd, 0x1, 0xc071353500, 0x7f57aae60080) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0a09a6cc0, 0xc1b68708a0, 0xc16ff49950, 0x2, 0x2, 0xc1117d3bc0, 0x2b44f80, 0xc009222e70, 0x657375040202cfa7, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x65726774736f7011, 0x74726f7078655f73) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc009222e40, 0xc081b915e0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10026061 [runnable]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc144aa3800, 0xc1ac66a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc2a35fec60, 0xc1ac66a000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc100be0270, 0xc1ac66a000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0d8222320, 0x7f56e2d6a100, 0xc100be0270, 0xc1ac66a000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0d8222320, 0x7f56e2d6a100, 0xc100be0270, 0xc1ac66a000, 0x8000, 0x8000, 0x2b6fc80, 0xc100be0270, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0f2165b00, 0x2b44ea0, 0xc0d8222320, 0xc1ac66a000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x30f30796f3527701, 0xb671529984dc05df, 0xc0741473c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x30f30796f3527701, 0xb671529984dc05df, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x30f30796f3527701, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc00335f400, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10895787 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1601d5450, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc13160b0e0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc157d7d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10903000 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0xc0c4d23000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc0c2630fe0, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00668f080, 0xc0fd9cdc80, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00668f080, 0xc06f4a9960, 0x2, 0x2, 0xc11efde300, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc00668f080, 0xc1817aa120, 0xc06f4a9960, 0x2, 0x2, 0xc11efde360, 0x2b44f80, 0xc089aa9860, 0x6c6a644561556f30, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc089aa9830, 0xc0830f1ce0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10938626 [select]: | |
| net/http.(*persistConn).readLoop(0xc0d965a7e0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10941541 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f31f7b00) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10942055 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc07e5b9f20, 0xc07e5b9f80, 0xc02599c060, 0xc0fdba1890, 0xc025f2bc50, 0xc0e0605810, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10939116 [select]: | |
| net/http.(*persistConn).writeLoop(0xc1003725a0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10938690 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0ca67d200) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10885628 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc153326480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10885567 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec64bd908, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1af554a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1af554a00, 0xc0bc3dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1af554a00, 0xc0bc3dc000, 0x8000, 0x8000, 0x203058, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc122508338, 0xc0bc3dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1b6424600, 0xc0bb26ae38, 0x9, 0x9, 0xc0c57da700, 0x1007f57aae89180, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1b6424600, 0xc0bb26ae38, 0x9, 0x9, 0x9, 0x601742d1, 0xc2aa962dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0bb26ae38, 0x9, 0x9, 0x2b41140, 0xc1b6424600, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0bb26ae00, 0xc147d2de40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc163466a80, 0xc198818720, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc163466a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc163466a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4976481 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e2c415f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc11fa22a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc11fa22a80, 0xc148d32000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc11fa22a80, 0xc148d32000, 0x8000, 0x8000, 0x203057, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc000fc4038, 0xc148d32000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1d2f468a0, 0xc07ba8e498, 0x9, 0x9, 0xc0dd272e00, 0x1007f57aae8a5d8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1d2f468a0, 0xc07ba8e498, 0x9, 0x9, 0x9, 0x601742e1, 0xc08f862dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc07ba8e498, 0x9, 0x9, 0x2b41140, 0xc1d2f468a0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc07ba8e460, 0xc1a94732c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1e3012180, 0xc1c1b065a0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1e3012180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1e3012180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10885627 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1b1911680, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1a311eae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc153326480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942931 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f57841dde18, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc17e02ee18, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc17e02ee00, 0xc07a7b8c00, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc17e02ee00, 0xc07a7b8c00, 0x109b, 0x109b, 0x203045, 0x3f99388, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc242704128, 0xc07a7b8c00, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc116a9e8e0, 0xc07a7b8c00, 0x109b, 0x109b, 0xc099350040, 0xc034437930, 0xc034437900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0f9a5b058, 0x2b41380, 0xc116a9e8e0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0f9a5ae00, 0x2b48520, 0xc242704128, 0x5, 0xc242704128, 0x216f320) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0f9a5ae00, 0x0, 0x0, 0x25) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0f9a5ae00, 0xc175efe000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f9db58c0, 0xc175efe000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc16794a6c0) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc16794a6c0, 0x1, 0xc210c36ba0, 0x0, 0x7701000000000091, 0x1268a21099145f4d, 0x3130b68face94e8e) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0f9db58c0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10942932 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f9db58c0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10811896 [select]: | |
| net/http.(*persistConn).writeLoop(0xc140d32480) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10792067 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e203ba40, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1381ea998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1381ea980, 0xc17b610000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1381ea980, 0xc17b610000, 0x8000, 0x8000, 0x203024, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc269754710, 0xc17b610000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc111f8fd40, 0xc05bdb21f8, 0x9, 0x9, 0xc071353500, 0x1007f57aae76ca0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc111f8fd40, 0xc05bdb21f8, 0x9, 0x9, 0x9, 0x601742d8, 0xc1796c3dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc05bdb21f8, 0x9, 0x9, 0x2b41140, 0xc111f8fd40, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc05bdb21c0, 0xc1fab7c8a0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc133bf6000, 0xc133bec060, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc133bf6000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc133bf6000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10940390 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1d7014660, 0xc1d70147e0, 0xc1d70148a0, 0xc0fd43dee0, 0xc0fefac2a0, 0xc0fcfe6b00, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10938629 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0df5ebe60) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10942784 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc165bac900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0b50e71d0, 0xc19d5868d0, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0b50e71d0, 0xc19d5868d0, 0x25, 0x38d89c7c, 0x7c, 0x38360dd04b628542, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0b50e71d0, 0xc19d5868d0, 0x25, 0x38d89c7c, 0x7c, 0x2bd18c0, 0xc0ae3e3d40, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0a49e9640, 0xc19d5868d0, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x7000102, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0a49e9640, 0xc19d5868d0, 0x25, 0x38d89c7c, 0x7c, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0a49e9640, 0xc1ff2103c0, 0x20, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0a49e9640, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0xc0f9741880, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0b50e71a0, 0xc09496ce10) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942799 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0f9db58c0, 0xc1a3782080, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0f85c7c00, 0xc16b120960, 0xc02af76678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0f85c7c00, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0f85c7c00, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc1a4c5e178, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0f85c7c00, 0x0, 0x0, 0x0, 0xc1a4c5e178, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0f85c7c00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc208a8ab40, 0xc208a8aba0, 0xc208a8ac00, 0xc11b38e670, 0xc129c7eba0, 0xc07ae6d340, 0x2b99d40, 0xc129c7eb40, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10942646 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc215b01bc0, 0xa, 0x2b99c80, 0xc0f83a0880, 0xc067b20630) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10940465 [select]: | |
| net/http.(*persistConn).readLoop(0xc0fc3b58c0) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10850785 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc171818ff0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc14a93ade0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0a5e3c780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4863112 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e2b76620, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc09e82b918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc09e82b900, 0xc13ab78000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc09e82b900, 0xc13ab78000, 0x8000, 0x8000, 0x800010601, 0x7f4e00000000, 0x106018f31fcc0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc182bf0a40, 0xc13ab78000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc10f5dad80, 0xc076a450d8, 0x9, 0x9, 0xc18f31fd40, 0x457243, 0xe7bcf5497fbb) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc10f5dad80, 0xc076a450d8, 0x9, 0x9, 0x9, 0xbffdee91964cedc6, 0x36ad08ea06f, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc076a450d8, 0x9, 0x9, 0x2b41140, 0xc10f5dad80, 0x0, 0x0, 0xc1baf20000, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc076a450a0, 0xc1baf20000, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1d99d4c00, 0xc008b13530, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1d99d4c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1d99d4c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4975786 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc07988d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10852987 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed32a4d48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc2ac2de698, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc2ac2de680, 0xc0cfb3a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc2ac2de680, 0xc0cfb3a000, 0x8000, 0x8000, 0x203028, 0x420bb5, 0xc0a256e020) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0bae50308, 0xc0cfb3a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1384fcd80, 0xc02d7709d8, 0x9, 0x9, 0xc01e1a1880, 0x1007f57aae8eca0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1384fcd80, 0xc02d7709d8, 0x9, 0x9, 0x9, 0x601742e4, 0xc079097dc8, 0x2ade866) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc02d7709d8, 0x9, 0x9, 0x2b41140, 0xc1384fcd80, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02d7709a0, 0xc01e25a000, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc25b9ccd80, 0xc1ac1d7bf0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc25b9ccd80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc25b9ccd80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10898359 [semacquire]: | |
| sync.runtime_Semacquire(0xc14c7a8160) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc14c7a8158) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc14c7a8150, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc0fae26cc0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc10ce43fb0, 0x255e040, 0x18, 0xc030def3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc16f7b86c0, 0x2bbccc0, 0xc214e99f60, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc16f7b86c0, 0x7f57841eead0, 0xc0093c8d80, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc16f7b86c0, 0x2bbbd60, 0xc0093c8d80, 0xc00083ae00, 0xc08e9e7688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc214e99ec0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc214e99ea0, 0xc214e99d40, 0x26b3908, 0xc214e99ea0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc214e99ea0, 0x2b99d40, 0xc10ce43e30) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc214e99e80, 0xc214e99d40, 0xc214e99d60, 0x7f57aae87a30, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc214e99e80, 0xc214e99d40, 0xc214e99d60, 0x2b99d01, 0xc214e99e80) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc214e99e80, 0x2b99d40, 0xc10ce43e00) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0xc214e99d40, 0xc214e99d80, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0x1, 0xc214e99d00) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0xc214e99d40, 0xc214e99da0, 0xa8e3442, 0xe7bec657db9b) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0x203085, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0xc214e99d40, 0xc214e99dc0, 0xc214e99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0xc4cfcf7c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc23ce89080, 0xc214e99d40, 0x26b3908, 0x2b99d40, 0xc10ce43da0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc125006d80, 0xc069f9cf00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc125006d80, 0xc069f9cf00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc108876de0, 0xc000c0cea0, 0x2bc6d40, 0xc125006d80, 0xc069f9cf00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 4975787 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec3d08b38, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00c582098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00c582080, 0xc1358ac000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00c582080, 0xc1358ac000, 0x8000, 0x8000, 0x20302e, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02ffae848, 0xc1358ac000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1a76a84e0, 0xc067482d58, 0x9, 0x9, 0xc00059ca80, 0x1007f57aae800f8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1a76a84e0, 0xc067482d58, 0x9, 0x9, 0x9, 0x601742dc, 0xc122740dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc067482d58, 0x9, 0x9, 0x2b41140, 0xc1a76a84e0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc067482d20, 0xc20dbea9a0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc07988d200, 0xc1c7fdf8c0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc07988d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc07988d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10893545 [semacquire]: | |
| sync.runtime_Semacquire(0xc0f99bdf90) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0f99bdf88) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0f99bdf80, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc14c71a120) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc12b66da70, 0x255e040, 0x18, 0xc0452dd3f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1645fa180, 0x2bbccc0, 0xc214ad3fa0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1645fa180, 0x7f57841eead0, 0xc00bbb1db0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1645fa180, 0x2bbbd60, 0xc00bbb1db0, 0xc00083ae00, 0xc08dcbb688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc214ad3f00, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc214ad3ee0, 0xc214ad3d80, 0x26b3908, 0xc214ad3ee0, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc214ad3ee0, 0x2b99d40, 0xc12b66d3e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc214ad3ec0, 0xc214ad3d80, 0xc214ad3da0, 0x7f57aae71180, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc214ad3ec0, 0xc214ad3d80, 0xc214ad3da0, 0x2b99d01, 0xc214ad3ec0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc214ad3ec0, 0x2b99d40, 0xc12b66d3b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0xc214ad3d80, 0xc214ad3dc0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0x1, 0xc214ad3d40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0xc214ad3d80, 0xc214ad3de0, 0x2c53561, 0xe7bebe8edcb3) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0x203085, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0xc214ad3d80, 0xc214ad3e00, 0xc214ad3d80, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0xc1132a6c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc190512a80, 0xc214ad3d80, 0x26b3908, 0x2b99d40, 0xc12b66d350) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00, 0xc177e63b00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00, 0xc177e63b00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc167faf1b0, 0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00, 0xc177e63b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10942645 [select]: | |
| net/http.(*persistConn).roundTrip(0xc20b290a20, 0xc0f83a0980, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0fa7c8900, 0xc147b2b080, 0xc0d7020678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0fa7c8900, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0fa7c8900, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc1bf178088, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0fa7c8900, 0x0, 0x0, 0x0, 0xc1bf178088, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0fa7c8900, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc215b01a40, 0xc215b01aa0, 0xc215b01b00, 0xc0fa2794d0, 0xc02b217f80, 0xc0fc5306e0, 0x2b99d40, 0xc02b217f20, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10942830 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc0e1516d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0b31d7e30, 0xc221e0fda0, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0b31d7e30, 0xc221e0fda0, 0x25, 0x38d89c7c, 0x7c, 0x4bf892b02a805d42, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0b31d7e30, 0xc221e0fda0, 0x25, 0x38d89c7c, 0x7c, 0x2bd18c0, 0xc0f9db5560, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0fa907d80, 0xc221e0fda0, 0x25, 0x38d89c7c, 0x7c, 0x0, 0x7000102, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0fa907d80, 0xc221e0fda0, 0x25, 0x38d89c7c, 0x7c, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0fa907d80, 0xc0d4d25ba0, 0x20, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0fa907d80, 0x38d89c7c, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0b31d7e00, 0xc0dc560750) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10851102 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783db0d70, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc009073998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc009073980, 0xc0cc522000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc009073980, 0xc0cc522000, 0x8000, 0x8000, 0x203011, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc59d9f81e0, 0xc0cc522000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc13123b0e0, 0xc07cafd7d8, 0x9, 0x9, 0xc0011f7880, 0x1007f57aae60080, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc13123b0e0, 0xc07cafd7d8, 0x9, 0x9, 0x9, 0x601742e4, 0xc00f1bbdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc07cafd7d8, 0x9, 0x9, 0x2b41140, 0xc13123b0e0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc07cafd7a0, 0xc085a9ffc0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc079683980, 0xc13310e8d0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc079683980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc079683980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10939048 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4f47fba288, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f9973a18, 0x72, 0xd700, 0xd701, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f9973a00, 0xc26bcba000, 0xd701, 0xd701, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f9973a00, 0xc26bcba000, 0xd701, 0xd701, 0x203040, 0x752c50, 0xc0e2345638) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0e08ca770, 0xc26bcba000, 0xd701, 0xd701, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc1006cd400, 0xc26bcba000, 0xd701, 0xd701, 0x1790, 0x96d7, 0xc1f8384e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0e2345758, 0x2b41380, 0xc1006cd400, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0e2345500, 0x2b48520, 0xc0e08ca770, 0x5, 0xc0e08ca770, 0x1780) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0e2345500, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0e2345500, 0xc1f663bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0df5ebe60, 0xc1f663bab2, 0x254e, 0x254e, 0x1780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc11d55d380, 0xc1f663bab2, 0x254e, 0x254e, 0x1780, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc1855f5720, 0xc1f663bab2, 0x254e, 0x254e, 0x1780, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0d7976000, 0xc1f663bab2, 0x254e, 0x254e, 0x1780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0d7976000, 0xc1f663bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0d7976040, 0xc1f663bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0d7976040, 0xc1f6636000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc01d1000c0, 0xc01d100360, 0xc01d100420, 0xc0f6cdc8f0, 0xc0ff7503c0, 0xc0e4372bb0, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10935180 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0deba5680, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc005533aa0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0e1c1bb00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 5014158 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec4700c28, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc176ca6998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc176ca6980, 0xc1b93e6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc176ca6980, 0xc1b93e6000, 0x8000, 0x8000, 0x20303f, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc17bb781d8, 0xc1b93e6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc177404780, 0xc02be90118, 0x9, 0x9, 0xc06f342700, 0x1007f57aae71848, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc177404780, 0xc02be90118, 0x9, 0x9, 0x9, 0x601742e4, 0xc16e7d2dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc02be90118, 0x9, 0x9, 0x2b41140, 0xc177404780, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02be900e0, 0xc1ec6c21e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1dff65380, 0xc177af6f30, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1dff65380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1dff65380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 5015586 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e151aed0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0ad811f18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0ad811f00, 0xc1d321a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0ad811f00, 0xc1d321a000, 0x8000, 0x8000, 0x203071, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0042e46b0, 0xc1d321a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1982d87e0, 0xc0e01ff0d8, 0x9, 0x9, 0xc0022a8380, 0x1007f57aae85848, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1982d87e0, 0xc0e01ff0d8, 0x9, 0x9, 0x9, 0x601742d7, 0xc182a01dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0e01ff0d8, 0x9, 0x9, 0x2b41140, 0xc1982d87e0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0e01ff0a0, 0xc19f2a8fa0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1cbf0fe00, 0xc1a4395500, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10885566 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc163466a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10026059 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc1661d9800, 0xc1f1f94000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc1d5aa8b40, 0xc1f1f94000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc0fccdec00, 0xc1f1f94000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0f19c0b90, 0x7f56e2d6a100, 0xc0fccdec00, 0xc1f1f94000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0f19c0b90, 0x7f56e2d6a100, 0xc0fccdec00, 0xc1f1f94000, 0x8000, 0x8000, 0x2b6fc80, 0xc0fccdec00, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1007aad00, 0x2b44ea0, 0xc0f19c0b90, 0xc1f1f94000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xb6cf86dc03457601, 0x998af2cd9a014ec4, 0xc19c6139c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xb6cf86dc03457601, 0x998af2cd9a014ec4, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xb6cf86dc03457601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0025e3800, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 5015550 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5784190818, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1a36e2298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1a36e2280, 0xc1d3318000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1a36e2280, 0xc1d3318000, 0x8000, 0x8000, 0x20302c, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0042e46e0, 0xc1d3318000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1183d4480, 0xc0f71761f8, 0x9, 0x9, 0xc0b02e5880, 0x1007f57aae8f368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1183d4480, 0xc0f71761f8, 0x9, 0x9, 0x9, 0x601742d6, 0xc04f7bbdc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0f71761f8, 0x9, 0x9, 0x2b41140, 0xc1183d4480, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0f71761c0, 0xc1fa970200, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1bf470780, 0xc1183955c0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1bf470780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1bf470780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 5072027 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc15b091720, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc21c6302a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc06362ed80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4864209 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1ce23d680) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10938515 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f5c88900) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10942832 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc1c5de4b40, 0xa, 0x2b99c80, 0xc0fa907e00, 0xc0b31d7f50) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10943026 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02346d0e0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10895439 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc165941200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4861636 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1b9453e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10861163 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0b0442c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10898139 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec5eca388, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc183e18498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc183e18480, 0xc0953e2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc183e18480, 0xc0953e2000, 0x8000, 0x8000, 0x203025, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc003cde2e0, 0xc0953e2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc17011c780, 0xc109629998, 0x9, 0x9, 0xc01e1a1880, 0x1007f57aae86ca0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc17011c780, 0xc109629998, 0x9, 0x9, 0x9, 0x601742cd, 0xc00b4f4dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc109629998, 0x9, 0x9, 0x2b41140, 0xc17011c780, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc109629960, 0xc1badb6e60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc165902f00, 0xc0f7418150, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc165902f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc165902f00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942831 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0f9db57a0, 0xc0fa907f00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0f85c7900, 0xc16b1204e0, 0xc00f3a0678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0f85c7900, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0f85c7900, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc1a4c5e168, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0f85c7900, 0x0, 0x0, 0x0, 0xc1a4c5e168, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0f85c7900, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc1c5de49c0, 0xc1c5de4a20, 0xc1c5de4a80, 0xc1a2d75fa0, 0xc0b31d7e90, 0xc0e0d9eb00, 0x2b99d40, 0xc0b31d7e30, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10885461 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc168f0bb30, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1b4dda180, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1656a8480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10938529 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f5c88b40) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4860374 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1cf891b00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942711 [select]: | |
| net/http.(*persistConn).writeLoop(0xc111440240) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4976061 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed0bf89e8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc02b612918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc02b612900, 0xc1042ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc02b612900, 0xc1042ce000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000000) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc15fd0a800, 0xc1042ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1ac2a6f60, 0xc10d4e5298, 0x9, 0x9, 0xc08f867d40, 0x457243, 0xe7bcf564a90b) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1ac2a6f60, 0xc10d4e5298, 0x9, 0x9, 0x9, 0xbffdee91965b127a, 0x36ad09cc522, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc10d4e5298, 0x9, 0x9, 0x2b41140, 0xc1ac2a6f60, 0x0, 0x0, 0xc1b91ee040, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc10d4e5260, 0xc1b91ee040, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0f7624180, 0xc06abc97a0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0f7624180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0f7624180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 4976060 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0f7624180) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10811895 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e203c140, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc10f9d4218, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc10f9d4200, 0xc000de5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc10f9d4200, 0xc000de5000, 0x1000, 0x1000, 0x4378cc, 0xc13e7f1b58, 0x4647d0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc19466c028, 0xc000de5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*persistConn).Read(0xc140d32480, 0xc000de5000, 0x1000, 0x1000, 0xc0c71a63c0, 0xc13e7f1c58, 0x405125) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc0de94d200) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc0de94d200, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc20ce265a0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc140d32480) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10938570 [select]: | |
| net/http.(*persistConn).readLoop(0xc0df5ebc20) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10938615 [select]: | |
| net/http.(*persistConn).readLoop(0xc0e809d560) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10813741 [select]: | |
| net/http.(*persistConn).writeLoop(0xc1495d6120) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10887531 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc143998900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 5014157 [select, 32 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1dff65380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4984224 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc2b2fb3040, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1982d8840, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1cbf0fe00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942901 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4ec30765c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0fad73218, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0fad73200, 0xc0ce5df300, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0fad73200, 0xc0ce5df300, 0x109b, 0x109b, 0x203027, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc845772098, 0xc0ce5df300, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc09c605fc0, 0xc0ce5df300, 0x109b, 0x109b, 0xc09c605cc0, 0xc002183968, 0xc002183900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0d4d11058, 0x2b41380, 0xc09c605fc0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0d4d10e00, 0x2b48520, 0xc845772098, 0x5, 0xc845772098, 0x3f6bae0) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0d4d10e00, 0x0, 0x0, 0x25) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0d4d10e00, 0xc1759aa000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0aaa58fc0, 0xc1759aa000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc16b121320) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc16b121320, 0x1, 0xc20a7f48a0, 0x0, 0x7701000000000091, 0x1268a21099145f4d, 0x3130b68face94e8e) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0aaa58fc0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10885555 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc175a1e050, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc13da1ec00, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc163466000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10942758 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc215d62ea0, 0xa, 0x2b99c80, 0xc0a9035a80, 0xc11a7d6210) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10942716 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc20d237560, 0xa, 0x2b99c80, 0xc1009eaf40, 0xc0b52e5c80) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 10938411 [runnable]: | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc026a4e000, 0xc026a4e1e0, 0xc026a4e240, 0xc0f8b2c0c0, 0xc0ffb9a3f0, 0xc0e86940b0, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:71 +0x16c | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4861637 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783d33900, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc139e9cf18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc139e9cf00, 0xc088662000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc139e9cf00, 0xc088662000, 0x8000, 0x8000, 0x203029, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc269754c78, 0xc088662000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc271157440, 0xc112d591b8, 0x9, 0x9, 0xc0dd272e00, 0x1007f57aae81f10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc271157440, 0xc112d591b8, 0x9, 0x9, 0x9, 0x601742d0, 0xc1f571ddc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc112d591b8, 0x9, 0x9, 0x2b41140, 0xc271157440, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc112d59180, 0xc2132192a0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1b9453e00, 0xc12fcb72f0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1b9453e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1b9453e00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10942882 [select]: | |
| github.com/minio/minio-go/v7.Client.newRetryTimer.func2(0xc210c36720, 0xa, 0x2b99c80, 0xc0a49e96c0, 0xc0b50e72f0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:79 +0x20c | |
| created by github.com/minio/minio-go/v7.Client.newRetryTimer | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/retry.go:70 +0x163 | |
| goroutine 4863110 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0778fe230, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc210887a40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1d99d4c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 4964423 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc1b4efe780) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4861509 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc06b2d7500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10943030 [select]: | |
| net/http.(*persistConn).writeLoop(0xc14c1f0fc0) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10941166 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f56e203c060, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f869f118, 0x72, 0xb800, 0xb87a, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f869f100, 0xc250490000, 0xb87a, 0xb87a, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f869f100, 0xc250490000, 0xb87a, 0xb87a, 0x203020, 0x752c50, 0xc0dabe8f38) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc2254da3d8, 0xc250490000, 0xb87a, 0xb87a, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc0807d00a0, 0xc250490000, 0xb87a, 0xb87a, 0x1b48, 0x7433, 0xc1f1a64e90) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0dabe9058, 0x2b41380, 0xc0807d00a0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0dabe8e00, 0x2b48520, 0xc2254da3d8, 0x5, 0xc2254da3d8, 0x1b38) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0dabe8e00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0dabe8e00, 0xc1ff84bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f5c88900, 0xc1ff84bab2, 0x254e, 0x254e, 0x1b38, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).Read(0xc1128475c0, 0xc1ff84bab2, 0x254e, 0x254e, 0x1b38, 0x0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:212 +0x141 | |
| io.(*LimitedReader).Read(0xc1f8982140, 0xc1ff84bab2, 0x254e, 0x254e, 0x1b38, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:451 +0x63 | |
| net/http.(*body).readLocked(0xc0fbe2e9c0, 0xc1ff84bab2, 0x254e, 0x254e, 0x1b38, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:850 +0x5f | |
| net/http.(*body).Read(0xc0fbe2e9c0, 0xc1ff84bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transfer.go:842 +0xf2 | |
| net/http.(*bodyEOFSignal).Read(0xc0fbe2ea00, 0xc1ff84bab2, 0x254e, 0x254e, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2710 +0xde | |
| io.ReadAtLeast(0x2b48600, 0xc0fbe2ea00, 0xc1ff846000, 0x8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc0c5ec4000, 0xc0c5ec4060, 0xc0c5ec40c0, 0xc0fcce8a80, 0xc0fe8503f0, 0xc0ccfa7e40, 0x2b99c80, 0xc000fa0080, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:195 +0xd7d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10885557 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed8bf8aa8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1af554998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1af554980, 0xc0b7ed6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1af554980, 0xc0b7ed6000, 0x8000, 0x8000, 0x20301a, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc122508330, 0xc0b7ed6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc13da1e060, 0xc0bb26ad58, 0x9, 0x9, 0xc03053e380, 0x1007f57aae71f10, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc13da1e060, 0xc0bb26ad58, 0x9, 0x9, 0x9, 0x601742e4, 0xc047e92dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0bb26ad58, 0x9, 0x9, 0x2b41140, 0xc13da1e060, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0bb26ad20, 0xc202ba7800, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc163466000, 0xc1a5cfdcb0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc163466000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc163466000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10026058 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc14c2d0a80, 0xc1a47e6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc13ca1f500, 0xc1a47e6000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc102c30b70, 0xc1a47e6000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0e754ae60, 0x7f56e2d6a100, 0xc102c30b70, 0xc1a47e6000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0e754ae60, 0x7f56e2d6a100, 0xc102c30b70, 0xc1a47e6000, 0x8000, 0x8000, 0x2b6fc80, 0xc102c30b70, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc1007aa700, 0x2b44ea0, 0xc0e754ae60, 0xc1a47e6000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0xa4a707eed5947601, 0x6e416921dd03320e, 0xc19ee20b40, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xa4a707eed5947601, 0x6e416921dd03320e, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0xa4a707eed5947601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc004d19500, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10902472 [semacquire]: | |
| sync.runtime_Semacquire(0xc0fccdefd0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0fccdefc8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0fccdefc0, 0x0, 0xc0fc4335b0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketChunkReader).preload(0xc1bbec55c0, 0xc0a7b58240, 0x2) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:2245 +0x260 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0059cb590, 0xc0b516ff00, 0xc1bbec55c0, 0xc0f619d0a0, 0x2, 0x2, 0xc1bbec42a0, 0x2b44f80, 0xc0fdbf24e0, 0x127cb350, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:758 +0xcb6 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x127ccd90, 0x127ccdf0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0fdbf24b0, 0xc082636e00) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 4864308 [select, 34 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc180f4b080) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10810366 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4edb4859b0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc15346da98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc15346da80, 0xc02a93d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc15346da80, 0xc02a93d000, 0x1000, 0x1000, 0x4db33a, 0xe7c69b5b6273, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1942fa090, 0xc02a93d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| net/http.(*connReader).Read(0xc23d7272c0, 0xc02a93d000, 0x1000, 0x1000, 0xbffdeeb36df2fa25, 0x38a56d532bf, 0x3f6bae0) | |
| /usr/local/go/src/net/http/server.go:797 +0xf4 | |
| bufio.(*Reader).fill(0xc02cf74180) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc02cf74180, 0x4, 0x38a56d532bf, 0x3f6bae0, 0x0, 0x0, 0x3f6bae0) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*conn).serve(0xc00b5a8320, 0x2b99c80, 0xc15cebf3c0) | |
| /usr/local/go/src/net/http/server.go:1949 +0x9d7 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:2962 +0x35c | |
| goroutine 10938628 [select]: | |
| net/http.(*persistConn).readLoop(0xc0df5ebe60) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10896903 [semacquire]: | |
| sync.runtime_Semacquire(0xc089aa9840) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc089aa9838) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc089aa9830, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc16a3c3860) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc089aa96b0, 0x255e040, 0x18, 0xc0c60d33f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc11efde360, 0x2bbccc0, 0xc01f018ec0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc11efde360, 0x7f57841eead0, 0xc06f4a8f40, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc11efde360, 0x2bbbd60, 0xc06f4a8f40, 0xc00083ae00, 0xc0c4d23688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc01f018e20, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc01f018e00, 0xc01f018ca0, 0x26b3908, 0xc01f018e00, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc01f018e00, 0x2b99d40, 0xc089aa9560) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc01f018de0, 0xc01f018ca0, 0xc01f018cc0, 0x7f57aae60080, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc01f018de0, 0xc01f018ca0, 0xc01f018cc0, 0x2b99d01, 0xc01f018de0) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc01f018de0, 0x2b99d40, 0xc089aa9530) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0xc01f018ca0, 0xc01f018ce0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0x1, 0xc01f018c60) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0xc01f018ca0, 0xc01f018d00, 0x2a3528e4, 0xe7bee5fed039) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0x203007, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0xc01f018ca0, 0xc01f018d20, 0xc01f018ca0, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0xc0017b8c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc14eb3f140, 0xc01f018ca0, 0x26b3908, 0x2b99d40, 0xc089aa94d0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc136b5df00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc136b5df00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc09ce7f0e0, 0xc000c0cea0, 0x2bc6d40, 0xc143998900, 0xc136b5df00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10886713 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ec5ec9ba8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1b579cc98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1b579cc80, 0xc032ef4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1b579cc80, 0xc032ef4000, 0x8000, 0x8000, 0x400000801, 0x1, 0x8) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1ee642178, 0xc032ef4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc0e24eb260, 0xc0679e8c78, 0x9, 0x9, 0xc0450d5020, 0xc079096df7, 0xc079096d80) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc0e24eb260, 0xc0679e8c78, 0x9, 0x9, 0x9, 0x406594, 0xc0c71e88a0, 0xc079096df7) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0679e8c78, 0x9, 0x9, 0x2b41140, 0xc0e24eb260, 0x0, 0xc100000000, 0x2124f40, 0xc19d377e50) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0679e8c40, 0xc19d377e40, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1064c9380, 0xc218411200, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1064c9380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1064c9380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10898657 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x100000235ea00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00bf6d4a0, 0xc1c7c79800, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:183 +0xb1 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc0de12d7d0, 0xc1af1b9500, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc117a15600, 0xc0c04fa360, 0x3, 0x3, 0xc02f787200, 0x205, 0x205, 0x60, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc117a15600, 0xc16a997f80, 0xc0c04fa360, 0x3, 0x3, 0xc146ff9f20, 0x2b44f80, 0xc1af1b9b60, 0x3f6bae0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc01c76ab60, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc1af1b98f0, 0xc01d9a0e00) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942494 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc2403b4300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0b52e5b60, 0xc1f5d58750, 0x25, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0b52e5b60, 0xc1f5d58750, 0x25, 0x327bf044, 0x64, 0x50a88eeb716338c4, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0b52e5b60, 0xc1f5d58750, 0x25, 0x327bf044, 0x64, 0x2bd18c0, 0xc14c1f0d80, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0cb47d580, 0xc1f5d58750, 0x25, 0x327bf044, 0x64, 0x0, 0x2000107, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0cb47d580, 0xc1f5d58750, 0x25, 0x327bf044, 0x64, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0cb47d580, 0xc1e816e2a0, 0x20, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0cb47d580, 0x327bf044, 0x64, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0b2ec4540, 0xc0cafe6510) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10940419 [select]: | |
| net/http.(*persistConn).readLoop(0xc0f67b1440) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10847490 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc222b540f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ae7b60c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc14a946000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10943028 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0f88f2b40) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4975790 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f57840addd0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc18dae5398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc18dae5380, 0xc13d7c2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc18dae5380, 0xc13d7c2000, 0x8000, 0x8000, 0x203042, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc02d1b20b8, 0xc13d7c2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1a76a8540, 0xc067482e38, 0x9, 0x9, 0xc01e1a1880, 0x1007f57aae825d8, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1a76a8540, 0xc067482e38, 0x9, 0x9, 0x9, 0x601742d7, 0xc00b4f1dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc067482e38, 0x9, 0x9, 0x2b41140, 0xc1a76a8540, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc067482e00, 0xc1fe2598e0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc07988d500, 0xc1ccfea030, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc07988d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc07988d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10940969 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc0cf6dfd40, 0x15, 0x7f3e16fe7905, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc10267d980, 0xc1013d2e00, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc10267d980, 0xc1026bc790, 0x2, 0x2, 0xc047f0e050, 0xd, 0x1, 0xc0b02e5880, 0x7f57aae825d8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc10267d980, 0xc1b042cea0, 0xc1026bc790, 0x2, 0x2, 0xc13531f260, 0x2b44f80, 0xc101475d70, 0xc0004066b8, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xc0be271bc0, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc101475d40, 0xc1026f51f0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10861164 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ef52c3a88, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc28dbc1e98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc28dbc1e80, 0xc117674000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc28dbc1e80, 0xc117674000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000000) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc2427040a0, 0xc117674000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc140a8de60, 0xc0c068b298, 0x9, 0x9, 0xc1132a2d40, 0x457243, 0xe7c38fc1c0f2) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc140a8de60, 0xc0c068b298, 0x9, 0x9, 0x9, 0xbffdee98abd81000, 0x3716b07daa7, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc0c068b298, 0x9, 0x9, 0x2b41140, 0xc140a8de60, 0x0, 0x0, 0xc1a6a23c00, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0c068b260, 0xc1a6a23c00, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0b0442c00, 0xc188fe9da0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0b0442c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10943029 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4eea141160, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0cd7ebb98, 0x72, 0x1000, 0x10fb, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0cd7ebb80, 0xc0cf9ab300, 0x10fb, 0x10fb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0cd7ebb80, 0xc0cf9ab300, 0x10fb, 0x10fb, 0x20301f, 0x2b415c0, 0xc00016c170) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1a55fa168, 0xc0cf9ab300, 0x10fb, 0x10fb, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc07f1b5fa0, 0xc0cf9ab300, 0x10fb, 0x10fb, 0x0, 0x2b484c0, 0xc120ac6900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc1880bfe58, 0x2b41380, 0xc07f1b5fa0, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc1880bfc00, 0x2b48520, 0xc1a55fa168, 0x5, 0xc1a55fa168, 0xc02c2e8001) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc1880bfc00, 0x0, 0x0, 0xc02c2e8000) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc1880bfc00, 0xc176862000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc14c1f0fc0, 0xc176862000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1a1673e00) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1a1673e00, 0x1, 0xc0f71325a0, 0xc000505cb8, 0x650bd3, 0xc149156db0, 0x216f320) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc14c1f0fc0) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 5005855 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc077acc2d0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0220e9aa0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc02dbc2000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10885565 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc175a1f7c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1b64246c0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc163466a80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10851101 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc079683980) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10942460 [select]: | |
| net/http.(*persistConn).roundTrip(0xc0ae1d9680, 0xc0a94c7d80, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0a2271c00, 0xc192db21e0, 0xc031f9e678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0a2271c00, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0a2271c00, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc02ffae338, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0a2271c00, 0x0, 0x0, 0x0, 0xc02ffae338, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0a2271c00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc208403f80, 0xc0cc190000, 0xc0cc190060, 0xc0b4027c50, 0xc0a040ea50, 0xc0c0a5ac60, 0x2b99d40, 0xc0a040e9f0, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 10852986 [select, 1 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc25b9ccd80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 5072028 [select, 31 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc06362ed80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 4860027 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed3cf6060, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc003cf8718, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc003cf8700, 0xc0895d0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc003cf8700, 0xc0895d0000, 0x8000, 0x8000, 0x20303d, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1040cc800, 0xc0895d0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc022c8e0c0, 0xc1271b2ff8, 0x9, 0x9, 0xc07aad6a80, 0x1007f57aae7b368, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc022c8e0c0, 0xc1271b2ff8, 0x9, 0x9, 0x9, 0x601742da, 0xc019fcadc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc1271b2ff8, 0x9, 0x9, 0x2b41140, 0xc022c8e0c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc1271b2fc0, 0xc01f093ae0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0ba5f7c80, 0xc1c9551cb0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc0ba5f7c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc0ba5f7c80) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10777743 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e203bb20, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1451d2398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1451d2380, 0xc01c73c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1451d2380, 0xc01c73c000, 0x8000, 0x8000, 0x4647d0, 0xc02b92f080, 0x4) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc14b9e4048, 0xc01c73c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc109adb7a0, 0xc165947298, 0x9, 0x9, 0xc0d41f9738, 0x461d40, 0xc0d41f96e0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc109adb7a0, 0xc165947298, 0x9, 0x9, 0x9, 0xc12c0c4d01, 0x10000000040cfa8, 0xc17bbfb730) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc165947298, 0x9, 0x9, 0x2b41140, 0xc109adb7a0, 0x0, 0x0, 0x0, 0xc17bbfb720) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc165947260, 0xc004070480, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc21722af00, 0xc1451ae870, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc21722af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc21722af00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10900977 [semacquire]: | |
| sync.runtime_Semacquire(0xc0fdbf24c0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0fdbf24b8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0fdbf24b0, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc19d9aeb40) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0fdbf2330, 0x255e040, 0x18, 0xc0408e53f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1bbec42a0, 0x2bbccc0, 0xc12a78e840, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1bbec42a0, 0x7f57841eead0, 0xc0f619c740, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1bbec42a0, 0x2bbbd60, 0xc0f619c740, 0xc00083ae00, 0xc0321c7688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc12a78e7a0, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc12a78e780, 0xc12a78e620, 0x26b3908, 0xc12a78e780, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc12a78e780, 0x2b99d40, 0xc0fdbf21b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc12a78e760, 0xc12a78e620, 0xc12a78e640, 0x7f57aae7e5d8, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc12a78e760, 0xc12a78e620, 0xc12a78e640, 0x2b99d01, 0xc12a78e760) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc12a78e760, 0x2b99d40, 0xc0fdbf2180) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0xc12a78e620, 0xc12a78e660, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0x1, 0xc12a78e5e0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0xc12a78e620, 0xc12a78e680, 0x2a1581ba, 0xe7bee5df2912) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0x20304a, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0xc12a78e620, 0xc12a78e6a0, 0xc12a78e620, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0xc0bb922c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc1c87a1080, 0xc12a78e620, 0x26b3908, 0x2b99d40, 0xc0fdbf2120) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc079683980, 0xc126a7fd00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc079683980, 0xc126a7fd00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc09cbb9b40, 0xc000c0cea0, 0x2bc6d40, 0xc079683980, 0xc126a7fd00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10938584 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0df5ebb00) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 5014156 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc1eaa54690, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc148bc1560, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc1dff65380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10901541 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x3f98000, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc09d305f20, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0a09a7a40, 0xc23e0b4b40, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0a09a7a40, 0xc06f4a83a0, 0x2, 0x2, 0xc11a79a300, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0a09a7a40, 0xc1b6870c60, 0xc06f4a83a0, 0x2, 0x2, 0xc1b6870900, 0x2b44f80, 0xc16532cba0, 0x2e73746e6567612e, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x73256369706f7405, 0x2d6b636f6c726568) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc16532cb70, 0xc082037880) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942715 [select]: | |
| net/http.(*persistConn).roundTrip(0xc14c1f0fc0, 0xc1009eb040, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/transport.go:2559 +0x763 | |
| net/http.(*Transport).roundTrip(0xc001022780, 0xc0c6e83100, 0xc0e59a6f60, 0xc0e2cb0678, 0x40cfa8) | |
| /usr/local/go/src/net/http/transport.go:582 +0xa88 | |
| net/http.(*Transport).RoundTrip(0xc001022780, 0xc0c6e83100, 0xc001022780, 0x0, 0x0) | |
| /usr/local/go/src/net/http/roundtrip.go:17 +0x35 | |
| net/http.send(0xc0c6e83100, 0x2b485c0, 0xc001022780, 0x0, 0x0, 0x0, 0xc0e5a3e0e8, 0x2537f0f, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:252 +0x43e | |
| net/http.(*Client).send(0xc000e87aa0, 0xc0c6e83100, 0x0, 0x0, 0x0, 0xc0e5a3e0e8, 0x0, 0x1, 0x0) | |
| /usr/local/go/src/net/http/client.go:176 +0xfa | |
| net/http.(*Client).do(0xc000e87aa0, 0xc0c6e83100, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/client.go:699 +0x44a | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:567 | |
| github.com/minio/minio-go/v7.Client.do(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:472 +0x50 | |
| github.com/minio/minio-go/v7.Client.executeMethod(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api.go:585 +0x409 | |
| github.com/minio/minio-go/v7.Client.getObject(0xc000410780, 0xc001024060, 0x1, 0xc0002ee580, 0x14, 0xc000a02e30, 0x10, 0x1, 0xc000e87aa0, 0xc001020ac0, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:622 +0x25b | |
| github.com/minio/minio-go/v7.Client.GetObject.func1(0xc20d237380, 0xc20d2373e0, 0xc20d237440, 0xc0ff149e20, 0xc0b52e5bc0, 0xc0d134ac60, 0x2b99d40, 0xc0b52e5b60, 0x7ffc6fe0408a, 0x1a, ...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:97 +0x32d | |
| created by github.com/minio/minio-go/v7.Client.GetObject | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:62 +0x346 | |
| goroutine 4864258 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e1c27fe8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0f5ff9718, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0f5ff9700, 0xc13ed9e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0f5ff9700, 0xc13ed9e000, 0x8000, 0x8000, 0x203044, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc182bf0e98, 0xc13ed9e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc12ef1a9c0, 0xc076a451b8, 0x9, 0x9, 0xc001595180, 0x1007f57aae5ec28, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc12ef1a9c0, 0xc076a451b8, 0x9, 0x9, 0x9, 0x601742e4, 0xc0997dadc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc076a451b8, 0x9, 0x9, 0x2b41140, 0xc12ef1a9c0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc076a45180, 0xc16baee8c0, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1d99d5380, 0xc1b9afd4d0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1d99d5380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1d99d5380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10026057 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc0c2a82000, 0xc1f5a4c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore.(*timingReadCloser).Read(0xc00171a2a0, 0xc1f5a4c000, 0x8000, 0x8000, 0x1, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:479 +0x55 | |
| github.com/thanos-io/thanos/pkg/objstore.(*tracingReadCloser).Read(0xc10121c0f0, 0xc1f5a4c000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:122 +0x58 | |
| io.copyBuffer(0x2b44ea0, 0xc0a8670230, 0x7f56e2d6a100, 0xc10121c0f0, 0xc1f5a4c000, 0x8000, 0x8000, 0x2187720, 0x2053200, 0x7f56e2d6a100) | |
| /usr/local/go/src/io/io.go:405 +0x122 | |
| io.CopyBuffer(0x2b44ea0, 0xc0a8670230, 0x7f56e2d6a100, 0xc10121c0f0, 0xc1f5a4c000, 0x8000, 0x8000, 0x2b6fc80, 0xc10121c0f0, 0x0) | |
| /usr/local/go/src/io/io.go:378 +0x82 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*chunkedIndexReader).CopySymbols(0xc0fae43f00, 0x2b44ea0, 0xc0a8670230, 0xc1f5a4c000, 0x8000, 0x8000, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:210 +0x1f4 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.WriteBinary(0x2b99c80, 0xc000fa0080, 0x7f5782c74008, 0xc000f99b60, 0x8b81aa63356b7601, 0x705428cddad64f6d, 0xc12125c7c0, 0x37, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go:92 +0x300 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.NewLazyBinaryReader(0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x8b81aa63356b7601, 0x705428cddad64f6d, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:108 +0x62e | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*ReaderPool).NewBinaryReader(0xc000fa0b00, 0x2b99c80, 0xc000fa0080, 0x2b42d60, 0xc000f99ad0, 0x7f5782c74008, 0xc000f99b60, 0xc000413ad0, 0xf, 0x8b81aa63356b7601, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/reader_pool.go:75 +0x10c | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).addBlock(0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc0021d7700, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:492 +0x4f7 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks.func1(0xc0d374efc0, 0xc0005f9e60, 0x2b99c80, 0xc000fa0080, 0xc04d8a7ea0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:379 +0x61 | |
| created by github.com/thanos-io/thanos/pkg/store.(*BucketStore).SyncBlocks | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:377 +0x15a | |
| goroutine 10936302 [select]: | |
| net/http.(*persistConn).writeLoop(0xc100372240) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 4859013 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f5783047e08, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00c583318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00c583300, 0xc090026000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc00c583300, 0xc090026000, 0x8000, 0x8000, 0x203014, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc25237e360, 0xc090026000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc208a122a0, 0xc03b89c1f8, 0x9, 0x9, 0xc002c80a80, 0x1007f57aae887c0, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc208a122a0, 0xc03b89c1f8, 0x9, 0x9, 0x9, 0x601742d1, 0xc019fc9dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc03b89c1f8, 0x9, 0x9, 0x2b41140, 0xc208a122a0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc03b89c1c0, 0xc141b37f60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc1b9453380, 0xc1cc3f49f0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc1b9453380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc1b9453380) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10901333 [semacquire]: | |
| sync.runtime_Semacquire(0xc02b217f00) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc02b217ef8) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc02b217ef0, 0xc0bcb563f0, 0x2b99c80) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0a09a6140, 0xc0fa553b80, 0x6, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1882 +0x863 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0a09a6140, 0xc16ff49950, 0x2, 0x2, 0xc0c77f8090, 0xd, 0x1, 0xc00169b180, 0x7f57aae7d180) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0076a5680, 0xc0a09a6140, 0xc00bc88f00, 0xc16ff49950, 0x2, 0x2, 0xc1117d3bc0, 0x2b44f80, 0xc009222e70, 0x61676974696d5f6f, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0xefc8afd87972616d, 0x6369706f74050201) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc009222e40, 0xc081b902a0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10893546 [semacquire]: | |
| sync.runtime_Semacquire(0xc0f99bdd50) | |
| /usr/local/go/src/runtime/sema.go:56 +0x42 | |
| sync.(*WaitGroup).Wait(0xc0f99bdd48) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x64 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0f99bdd40, 0x0, 0x2b4a640) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31 | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func4(0x2b99d40, 0xc4650d3020) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:993 +0x33 | |
| github.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x2b99d40, 0xc0f99bdbc0, 0x255e040, 0x18, 0xc0c9d333f8, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/tracing/tracing.go:72 +0xcc | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc0005f9e60, 0xc1a1cb47e0, 0x2bbccc0, 0xc214e10020, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:992 +0xae3 | |
| github.com/cortexproject/cortex/pkg/storegateway.(*BucketStores).Series(0xc0001cf600, 0xc1a1cb47e0, 0x7f57841eead0, 0xc14ac6e5f0, 0x0, 0x0) | |
| /__w/cortex/cortex/pkg/storegateway/bucket_stores.go:245 +0x1da | |
| github.com/cortexproject/cortex/pkg/storegateway.(*StoreGateway).Series(0xc00083ae00, 0xc1a1cb47e0, 0x2bbbd60, 0xc14ac6e5f0, 0xc00083ae00, 0xc0ffdff688) | |
| /__w/cortex/cortex/pkg/storegateway/gateway.go:320 +0x82 | |
| github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb._StoreGateway_Series_Handler(0x246b9e0, 0xc00083ae00, 0x2bb8400, 0xc18995ff80, 0x24321a0, 0xc000776600) | |
| /__w/cortex/cortex/pkg/storegateway/storegatewaypb/gateway.pb.go:174 +0x109 | |
| github.com/cortexproject/cortex/pkg/cortex.ThanosTracerStreamInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc18995ff60, 0xc18995fe00, 0x26b3908, 0xc18995ff60, 0x23b6440) | |
| /__w/cortex/cortex/pkg/cortex/tracing.go:20 +0x10e | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb8700, 0xc18995ff60, 0x2b99d40, 0xc0f99bda40) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc18995ff40, 0xc18995fe00, 0xc18995fe20, 0x7f57aae807c0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe | |
| github.com/cortexproject/cortex/pkg/util/fakeauth.SetupAuthMiddleware.func2(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc18995ff40, 0xc18995fe00, 0xc18995fe20, 0x2b99d01, 0xc18995ff40) | |
| /__w/cortex/cortex/pkg/util/fakeauth/fake_auth.go:35 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb69c0, 0xc18995ff40, 0x2b99d40, 0xc0f99bda10) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0xc18995fe00, 0xc18995fe40, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x355 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0x1, 0xc18995fdc0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0xc18995fe00, 0xc18995fe60, 0x398e709c, 0xe7beb9bd4e54) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0x203062, 0x8701ba) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2bca9c0, 0xc00039bfb0, 0x40c700, 0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0xc18995fe00, 0xc18995fe80, 0xc18995fe00, ...) | |
| /__w/cortex/cortex/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0xc067334c68, 0x40cfa8) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x246b9e0, 0xc00083ae00, 0x2bb7020, 0xc02fafa780, 0xc18995fe00, 0x26b3908, 0x2b99d40, 0xc0f99bd9b0) | |
| /__w/cortex/cortex/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00, 0xc177e63c00, 0xc000dfcc00, 0x3f3f8a0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1336 +0x511 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00, 0xc177e63c00, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:1409 +0xc66 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc167faf1b0, 0xc000c0cea0, 0x2bc6d40, 0xc1cbf0fe00, 0xc177e63c00) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:746 +0xa1 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:744 +0xa1 | |
| goroutine 10899685 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d51c, 0x3f75900, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00bf6d518) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc00bf6d4a0, 0xc0cbb4cfa0, 0x15, 0x7f3e16fe7905, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc00c189f00, 0xc0f9d59500, 0x5, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00c189f00, 0xc1c6d5e900, 0x2, 0x2, 0x216f320, 0x3f99388, 0x0, 0x0, 0xbffdee9b1477ae36) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc00c189f00, 0xc18851eae0, 0xc1c6d5e900, 0x2, 0x2, 0xc1a1cb47e0, 0x2b44f80, 0xc0f99bdda0, 0xc0005d4428, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0f99bdd40, 0xc02624fb20) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10942458 [chan receive]: | |
| github.com/minio/minio-go/v7.(*Object).doGetRequest(...) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:276 | |
| github.com/minio/minio-go/v7.(*Object).Read(0xc02f9c7380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/minio/minio-go/v7/api-get-object.go:350 +0x227 | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).getRange(0xc0001b8fa0, 0x2b99d40, 0xc0b53e8360, 0xc142851860, 0x25, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:379 +0x20c | |
| github.com/thanos-io/thanos/pkg/objstore/s3.(*Bucket).GetRange(0xc0001b8fa0, 0x2b99d40, 0xc0b53e8360, 0xc142851860, 0x25, 0x327bf044, 0x64, 0x6577936a20529752, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/s3/s3.go:396 +0x77 | |
| github.com/thanos-io/thanos/pkg/objstore.(*metricBucket).GetRange(0xc0002fe780, 0x2b99d40, 0xc0b53e8360, 0xc142851860, 0x25, 0x327bf044, 0x64, 0x2bd18c0, 0xc0aaa58d80, 0x2b99d40, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/objstore.go:364 +0xe9 | |
| github.com/thanos-io/thanos/pkg/objstore.TracingBucket.GetRange(0x2bca540, 0xc0002fe780, 0x2b99c80, 0xc0a94c7c00, 0xc142851860, 0x25, 0x327bf044, 0x64, 0x0, 0x6000103, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/objstore/tracing.go:50 +0x23b | |
| github.com/thanos-io/thanos/pkg/store/cache.(*CachingBucket).GetRange(0xc0005c2720, 0x2b99c80, 0xc0a94c7c00, 0xc142851860, 0x25, 0x327bf044, 0x64, 0x1268a21099145f4d, 0x3130b68face94e8e, 0x533435594e365845, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/cache/caching_bucket.go:273 +0x23a | |
| github.com/cortexproject/cortex/pkg/storage/bucket.(*UserBucketReaderClient).GetRange(0xc000f99b60, 0x2b99c80, 0xc0a94c7c00, 0xc1f69f9720, 0x20, 0x327bf044, 0x64, 0x0, 0x0, 0x0, ...) | |
| /__w/cortex/cortex/pkg/storage/bucket/user_bucket_client.go:73 +0x13d | |
| github.com/thanos-io/thanos/pkg/store.(*bucketBlock).readIndexRange(0xc00354be00, 0x2b99c80, 0xc0a94c7c00, 0x327bf044, 0x64, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1441 +0xe7 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings.func3(0x43e216, 0x26b82e8) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1820 +0xf6 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0a040e9c0, 0xc0897a0a20) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10887532 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f51559ea7a0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc1430c8618, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc1430c8600, 0xc04e416000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc1430c8600, 0xc04e416000, 0x8000, 0x8000, 0x800010601, 0x0, 0x1060100000000) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc005fa44b8, 0xc04e416000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc149d083c0, 0xc037bf3a78, 0x9, 0x9, 0xc067331d40, 0x457243, 0xe7bf1539203e) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc149d083c0, 0xc037bf3a78, 0x9, 0x9, 0x9, 0xbffdee93aa0bb7d2, 0x36cc123ba6c, 0x3f6bae0) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc037bf3a78, 0x9, 0x9, 0x2b41140, 0xc149d083c0, 0x0, 0x0, 0xc08722c040, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc037bf3a40, 0xc08722c040, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc143998900, 0xc1807cab40, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc143998900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc143998900) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10901912 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00bf6d520, 0x0, 0x0) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*RWMutex).Lock(0xc00bf6d518) | |
| /usr/local/go/src/sync/rwmutex.go:103 +0x88 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc00bf6d4a0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).LabelValues(0xc00bf6d4a0, 0xc0a69f8600, 0x15, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:183 +0xb1 | |
| github.com/thanos-io/thanos/pkg/store.toPostingGroup(0xc101d317d0, 0xc06fffc9c0, 0x7080be, 0x2, 0xc03c9cfb10) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1713 +0x403 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc00b6e0000, 0xc16cee7820, 0x2, 0x2, 0xc00dbc7140, 0xd, 0x1, 0xc071353500, 0x7f57aae75848) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1563 +0x1a7 | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc00933edb0, 0xc00b6e0000, 0xc0efedb980, 0xc16cee7820, 0x2, 0x2, 0xc0080a1c80, 0x2b44f80, 0xc06fffcb10, 0x7563797365677167, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x365a354572676e4c, 0x51616e6a44764368) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc06fffcae0, 0xc0819f6ee0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66 | |
| goroutine 10894852 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc165d4d500) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10792066 [select, 2 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc133bf6000) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6 | |
| goroutine 10885629 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f4ed4a22120, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc10f8b4318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc10f8b4300, 0xc09a2f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc10f8b4300, 0xc09a2f4000, 0x8000, 0x8000, 0x20302f, 0x20302f, 0x20302f) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc1bf178660, 0xc09a2f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc19d29bd40, 0xc076bf77d8, 0x9, 0x9, 0x7f57aae5de98, 0x0, 0xc07625bda8) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc19d29bd40, 0xc076bf77d8, 0x9, 0x9, 0x9, 0xc0bc766080, 0xc0bc766001, 0xc07625bd01) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc076bf77d8, 0x9, 0x9, 0x2b41140, 0xc19d29bd40, 0x0, 0xc000000000, 0xc0e4dedf01, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc076bf77a0, 0xc0e4dedf80, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc153326480, 0xc1afa7fe60, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc153326480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc153326480) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10895440 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f56e2c409b0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc16d373a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc16d373a00, 0xc078eee000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc16d373a00, 0xc078eee000, 0x8000, 0x8000, 0x203053, 0x0, 0x60100000008) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc353b7c2a8, 0xc078eee000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| bufio.(*Reader).Read(0xc1496dfaa0, 0xc085ad9998, 0x9, 0x9, 0xc079c94000, 0x1007f57aae8fa30, 0x0) | |
| /usr/local/go/src/bufio/bufio.go:226 +0x24f | |
| io.ReadAtLeast(0x2b41140, 0xc1496dfaa0, 0xc085ad9998, 0x9, 0x9, 0x9, 0x601742d5, 0xc0006d6dc8, 0x4db7f6) | |
| /usr/local/go/src/io/io.go:310 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:329 | |
| golang.org/x/net/http2.readFrameHeader(0xc085ad9998, 0x9, 0x9, 0x2b41140, 0xc1496dfaa0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:237 +0x87 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc085ad9960, 0xc0c96beb60, 0x3f6bae0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/golang.org/x/net/http2/frame.go:492 +0xa1 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc165941200, 0xc16b6be7e0, 0x26b66b0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000c0cea0, 0x2bc6d40, 0xc165941200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:742 +0xdb | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000c0cea0, 0x2bc6d40, 0xc165941200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:703 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/server.go:702 +0x4fd | |
| goroutine 10943027 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f4ec3de3790, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:203 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0cda27c98, 0x72, 0x1000, 0x109b, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0cda27c80, 0xc0cda34000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b | |
| net.(*netFD).Read(0xc0cda27c80, 0xc0cda34000, 0x109b, 0x109b, 0x203040, 0x2b415c0, 0xc00016c170) | |
| /usr/local/go/src/net/fd_unix.go:202 +0x4f | |
| net.(*conn).Read(0xc0dd632030, 0xc0cda34000, 0x109b, 0x109b, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:184 +0x8e | |
| crypto/tls.(*atLeastReader).Read(0xc1006e2c60, 0xc0cda34000, 0x109b, 0x109b, 0x0, 0x2b484c0, 0xc0997db900) | |
| /usr/local/go/src/crypto/tls/conn.go:760 +0x60 | |
| bytes.(*Buffer).ReadFrom(0xc0c92ea258, 0x2b41380, 0xc1006e2c60, 0x40a685, 0x21a56c0, 0x2489b40) | |
| /usr/local/go/src/bytes/buffer.go:204 +0xb1 | |
| crypto/tls.(*Conn).readFromUntil(0xc0c92ea000, 0x2b48520, 0xc0dd632030, 0x5, 0xc0dd632030, 0xc233a8f501) | |
| /usr/local/go/src/crypto/tls/conn.go:782 +0xec | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0c92ea000, 0x0, 0x0, 0xc233a8f500) | |
| /usr/local/go/src/crypto/tls/conn.go:589 +0x115 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:557 | |
| crypto/tls.(*Conn).Read(0xc0c92ea000, 0xc16ad6e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:1233 +0x15b | |
| net/http.(*persistConn).Read(0xc0f88f2b40, 0xc16ad6e000, 0x1000, 0x1000, 0x40505d, 0x60, 0x0) | |
| /usr/local/go/src/net/http/transport.go:1884 +0x75 | |
| bufio.(*Reader).fill(0xc1a1673c20) | |
| /usr/local/go/src/bufio/bufio.go:100 +0x103 | |
| bufio.(*Reader).Peek(0xc1a1673c20, 0x1, 0xc100d56240, 0xc0be237cb8, 0x650bd3, 0xc089aa96b0, 0x216f320) | |
| /usr/local/go/src/bufio/bufio.go:138 +0x4f | |
| net/http.(*persistConn).readLoop(0xc0f88f2b40) | |
| /usr/local/go/src/net/http/transport.go:2037 +0x1a8 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 4975785 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0f52728c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc1ac0a8060, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc07988d200) | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /__w/cortex/cortex/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1 | |
| goroutine 10938549 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0d872d560) | |
| /usr/local/go/src/net/http/transport.go:2336 +0x11c | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1707 +0xc7b | |
| goroutine 10938689 [select]: | |
| net/http.(*persistConn).readLoop(0xc0ca67d200) | |
| /usr/local/go/src/net/http/transport.go:2158 +0x9a0 | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1706 +0xc56 | |
| goroutine 10899920 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc0044c330c, 0x0, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc0044c3308) | |
| /usr/local/go/src/sync/mutex.go:138 +0xfc | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| sync.(*RWMutex).Lock(0xc0044c3308) | |
| /usr/local/go/src/sync/rwmutex.go:98 +0x97 | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).load(0xc0044c3290, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:218 +0xae | |
| github.com/thanos-io/thanos/pkg/block/indexheader.(*LazyBinaryReader).PostingsOffset(0xc0044c3290, 0xc0de31e640, 0x15, 0x7f4eae4a9bc1, 0x14, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/lazy_binary_reader.go:157 +0xa5 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).fetchPostings(0xc0056a8780, 0xc23c458900, 0x4, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1783 +0xb01 | |
| github.com/thanos-io/thanos/pkg/store.(*bucketIndexReader).ExpandedPostings(0xc0056a8780, 0xc00c6be840, 0x2, 0x2, 0xc16f7b8000, 0x253113a, 0x3, 0x2bb7920, 0x0) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:1601 +0x5ac | |
| github.com/thanos-io/thanos/pkg/store.blockSeries(0xc0071ab4d0, 0xc0056a8780, 0xc1224409c0, 0xc00c6be840, 0x2, 0x2, 0xc0dd83a2a0, 0x2b44f80, 0xc149156f90, 0x616e6f6974636173, ...) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:687 +0x6a | |
| github.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func2(0x6f74616369726261, 0x50201c0e5b8d872) | |
| /__w/cortex/cortex/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go:938 +0xbe | |
| golang.org/x/sync/errgroup.(*Group).Go.func1(0xc149156f60, 0xc02d4681c0) | |
| /__w/cortex/cortex/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59 | |
| created by golang.org/x/sync/errgroup.(*Group).Go |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment