Skip to content

Instantly share code, notes, and snippets.

@magnusja

magnusja/gist:f0cec405ace330ce8a25047646cbd0e5

Created November 26, 2020 10:36
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save magnusja/f0cec405ace330ce8a25047646cbd0e5 to your computer and use it in GitHub Desktop.
Save magnusja/f0cec405ace330ce8a25047646cbd0e5 to your computer and use it in GitHub Desktop.
Download ZIP
ray memory nightly nov 24 gc collect
Raw
gistfile1.txt
root@training-document-classification-8cfdt:/training# ray memory
2020-11-25 11:06:20,210 INFO scripts.py:1342 -- Connecting to Ray instance at 10.42.0.174:6379.
2020-11-25 11:06:20,210 INFO worker.py:654 -- Connecting to existing Ray cluster at address: 10.42.0.174:6379
-----------------------------------------------------------------------------------------------------
Object ID Reference Type Object Size Reference Creation Site
=====================================================================================================
; worker pid=464
ffffffffffffffffffffffff0100000001000000 PINNED_IN_MEMORY 18808561 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000002000000 PINNED_IN_MEMORY 5988172 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
; driver pid=6
3abfa8774e37914924ca2d6d0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
1b9f9f92dbc42766ff22d3fc0100000001000000 LOCAL_REFERENCE 42199077 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
6750fb6e5728c548501ed5620100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391
362823fde0070a4b8db8395b0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
cdeac006f660498818b034100100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
3f1fd21bac65529535ca5eb10100000001000000 LOCAL_REFERENCE 42199081 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
101bf5542ce052ed65df6f6a0100000001000000 LOCAL_REFERENCE 42199082 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
ffffffffffffffffffffffff0100000002000000 LOCAL_REFERENCE 5988172 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:364 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489
ffffffffffffffff84865f460100000001000000 USED_BY_PENDING_TASK ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_setup_remote_runner:233 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_start_trial:287 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:start_trial:344
5837fe3212a0f45284865f460100000001000000 LOCAL_REFERENCE ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_train:254 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:continue_training:377 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_runner.py:_execute_action:751
a31abd3e069e345ce4094a560100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
da8041c58cc4c5456bd238fa0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
d1339b52650f20e3ae935fc00100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
ffffffffffffffffffffffff0100000001000000 LOCAL_REFERENCE 18808561 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:363 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489
000c21fb760652ca6bd238fa0100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391
bc4ce58d94c1adbe82be70da0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
4305340a38238760ad5fa5cb0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
e804d335f4f03d7ad6a2003a0100000001000000 LOCAL_REFERENCE 2337 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
35d7a3b94b8ab51699506fd40100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
37681aee48c9017301d26efb0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
c490e45e54a5f21035ca5eb10100000001000000 LOCAL_REFERENCE 42199081 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391
62223d85d5e7cd76df5a1a820100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
3edfd0ab4d03f4e1501ed5620100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
b19fee1fe487d3337bbd90280100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
814c5aa2dda6577435ca5eb10100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
e1083dbe957d9fb7b9c4a7c60100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
6fdb0c3ff564615eb5dec6180100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
205ffc57cc94b7ff96494bee0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
671eed44decf009e0d247c270100000001000000 LOCAL_REFERENCE 42199030 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
2b8407b1b57ff42365df6f6a0100000001000000 LOCAL_REFERENCE 42199082 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391
c662ca5a11ff42730d247c270100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
1211da137a2627cddb6122010100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
93df71baff88c4c8b5dec6180100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
e221fcd97f554e8e65df6f6a0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
8ce7ce902fec182cf16d5bfb0100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
7d269126652173d084b65a940100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
b66c51bcf3976d42ff22d3fc0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
b1b09d76c3c9a836501ed5620100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:365 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489
b7603b6cc81d46c188866c7d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
1d7678d52cf5ccfedb6122010100000001000000 LOCAL_REFERENCE 42199082 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
36794f36fcf113360a5d5c950100000001000000 LOCAL_REFERENCE 2062 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
9b9842a4eee323f2cba888e30100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
90e2cc402c650603b5dec6180100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:470 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:391
b299cffadddbb85a3d80c2fe0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
84cdc86d23a9ae806bd238fa0100000001000000 LOCAL_REFERENCE 42199080 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:679 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:119 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:388
c65d7ae092f0e8c1d072aa390100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:324 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:367
-----------------------------------------------------------------------------------------------------
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment