Created
November 26, 2020 10:36
-
-
Save magnusja/f0cec405ace330ce8a25047646cbd0e5 to your computer and use it in GitHub Desktop.
ray memory nightly nov 24 gc collect
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
root@training-document-classification-8cfdt:/training# ray memory | |
2020-11-25 11:06:20,210 INFO scripts.py:1342 -- Connecting to Ray instance at 10.42.0.174:6379. | |
2020-11-25 11:06:20,210 INFO worker.py:654 -- Connecting to existing Ray cluster at address: 10.42.0.174:6379 | |
----------------------------------------------------------------------------------------------------- | |
Object ID Reference Type Object Size Reference Creation Site | |
===================================================================================================== | |
; worker pid=464 | |
ffffffffffffffffffffffff0100000001000000 PINNED_IN_MEMORY 18808561 (deserialize actor creation task arg) abc.DynamicTrainable.__init__ | |
ffffffffffffffffffffffff0100000002000000 PINNED_IN_MEMORY 5988172 (deserialize actor creation task arg) abc.DynamicTrainable.__init__ | |
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (deserialize actor creation task arg) abc.DynamicTrainable.__init__ | |
; driver pid=6 | |
3abfa8774e37914924ca2d6d0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
1b9f9f92dbc42766ff22d3fc0100000001000000 LOCAL_REFERENCE 42199077 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
6750fb6e5728c548501ed5620100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391 | |
362823fde0070a4b8db8395b0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
cdeac006f660498818b034100100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
3f1fd21bac65529535ca5eb10100000001000000 LOCAL_REFERENCE 42199081 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
101bf5542ce052ed65df6f6a0100000001000000 LOCAL_REFERENCE 42199082 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
ffffffffffffffffffffffff0100000002000000 LOCAL_REFERENCE 5988172 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:364 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489 | |
ffffffffffffffff84865f460100000001000000 USED_BY_PENDING_TASK ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_setup_remote_runner:233 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_start_trial:287 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:start_trial:344 | |
5837fe3212a0f45284865f460100000001000000 LOCAL_REFERENCE ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_train:254 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:continue_training:377 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_runner.py:_execute_action:751 | |
a31abd3e069e345ce4094a560100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
da8041c58cc4c5456bd238fa0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
d1339b52650f20e3ae935fc00100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
ffffffffffffffffffffffff0100000001000000 LOCAL_REFERENCE 18808561 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:363 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489 | |
000c21fb760652ca6bd238fa0100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391 | |
bc4ce58d94c1adbe82be70da0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
4305340a38238760ad5fa5cb0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
e804d335f4f03d7ad6a2003a0100000001000000 LOCAL_REFERENCE 2337 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
35d7a3b94b8ab51699506fd40100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
37681aee48c9017301d26efb0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
c490e45e54a5f21035ca5eb10100000001000000 LOCAL_REFERENCE 42199081 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391 | |
62223d85d5e7cd76df5a1a820100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
3edfd0ab4d03f4e1501ed5620100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
b19fee1fe487d3337bbd90280100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
814c5aa2dda6577435ca5eb10100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
e1083dbe957d9fb7b9c4a7c60100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
6fdb0c3ff564615eb5dec6180100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
205ffc57cc94b7ff96494bee0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
671eed44decf009e0d247c270100000001000000 LOCAL_REFERENCE 42199030 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
2b8407b1b57ff42365df6f6a0100000001000000 LOCAL_REFERENCE 42199082 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391 | |
c662ca5a11ff42730d247c270100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
1211da137a2627cddb6122010100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
93df71baff88c4c8b5dec6180100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
e221fcd97f554e8e65df6f6a0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
8ce7ce902fec182cf16d5bfb0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
7d269126652173d084b65a940100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
b66c51bcf3976d42ff22d3fc0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
b1b09d76c3c9a836501ed5620100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:365 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489 | |
b7603b6cc81d46c188866c7d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
1d7678d52cf5ccfedb6122010100000001000000 LOCAL_REFERENCE 42199082 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
36794f36fcf113360a5d5c950100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
9b9842a4eee323f2cba888e30100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
90e2cc402c650603b5dec6180100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391 | |
b299cffadddbb85a3d80c2fe0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
84cdc86d23a9ae806bd238fa0100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388 | |
c65d7ae092f0e8c1d072aa390100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367 | |
----------------------------------------------------------------------------------------------------- |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment