Skip to content

Instantly share code, notes, and snippets.

@magnusja
Created November 26, 2020 10:37
Show Gist options
  • Save magnusja/bfd4c861152b1446624e3cae9bfc0305 to your computer and use it in GitHub Desktop.
Save magnusja/bfd4c861152b1446624e3cae9bfc0305 to your computer and use it in GitHub Desktop.
ray memory 1.0.1 gc collect
root@training-document-classification-pvndc:/training#
C:\Users\xgrxjah>kubectl exec -i -t -n staging training-document-classification-pvndc -c training "--" sh -c "clear; (bash || ash || sh)"
root@training-document-classification-pvndc:/training# ray memory
2020-11-26 10:11:42,978 INFO scripts.py:1340 -- Connecting to Ray instance at 10.42.0.177:6379.
2020-11-26 10:11:42,979 INFO worker.py:650 -- Connecting to existing Ray cluster at address: 10.42.0.177:6379
-----------------------------------------------------------------------------------------------------
Object ID Reference Type Object Size Reference Creation Site
=====================================================================================================
; worker pid=223743
ffffffffffffffffffffffff0100000001000000 PINNED_IN_MEMORY 18808561 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000002000000 PINNED_IN_MEMORY 5988152 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
; worker pid=223231
ffffffffffffffffffffffff0100000001000000 PINNED_IN_MEMORY 18808561 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000002000000 PINNED_IN_MEMORY 5988152 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
; worker pid=222765
ffffffffffffffffffffffff0100000002000000 PINNED_IN_MEMORY 5988152 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000001000000 PINNED_IN_MEMORY 18808561 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (deserialize actor creation task arg) abc.DynamicTrainable.__init__
; driver pid=6
2dc2abf67a9e9c15520f0bfa0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
2a7eac90067da6d59baf422f0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
098ec8681ead2ccfec4e47360100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
8b63d9d026a35e4bd0c8e0150100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
c09ba83bb6fbb79a8fa7f3200100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
6fdf1d78f3c6e6415d3091fa0100000001000000 LOCAL_REFERENCE ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_train:254 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:continue_training:382 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_runner.py:_process_trial_restore:889
971055c1d00a6964820340560100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
9610bc55515d47a631ce37cb0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
e94c52e18e2e3988abf2aa230100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
dc54215a3525a432936ead920100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
f8b384057e585bd3ed9817f40100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ae752923407048ce9d6818d10100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
46b3afc0b045cdca537912cd0100000001000000 LOCAL_REFERENCE 42199081 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
a55ebf060d1a9022f9dfae570100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
13c8c9e1b945eb82dc42caf10100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
739f8ea7aa53e14f3237bd020100000001000000 LOCAL_REFERENCE 42199078 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
a0d55b90fdb020f6d7cfe0400100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
f677aa555d253c41bbfbae0d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
c09d4298384a3d4195e0a3050100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
7e305c80b5e163ac0cca1be00100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ffffffffffffffff5d3091fa0100000001000000 USED_BY_PENDING_TASK ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_setup_remote_runner:233 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_start_trial:287 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:start_trial:348
251ff06dd4d8db759a58f5ea0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
d7b8c8cf6a1c1bcc9dee0bed0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ffffffffffffffffffffffff0100000001000000 LOCAL_REFERENCE 18808561 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:363 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489
1c1f78cfa1452fa1b4eff96a0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
6735608f2babb8179d6818d10100000001000000 LOCAL_REFERENCE 42199074 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
210aa82c81f35ae7e92e43520100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
541bc43948f8363015bba0530100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
b3ca27510f89970697ddfcef0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
fdb6c2db35fa16a28b8a44d10100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
4f0f4548c4a53b32b36422730100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
d0695913f6672fecd277a6c20100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
e66b8af061df35f19a8e9f000100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
50a51437f2364893b9b08a2a0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
a74ad41e42c35082c3ea1f390100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
55257ee3b9161804369405020100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
4b321fba6cf5b94ff6815d9d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
124b52a9dc2ea468b8129f0f0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
8a23669bea903598bdbd5f440100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
a48f2c0245d5d536aa62673e0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
b634cadaaa35beec537912cd0100000001000000 LOCAL_REFERENCE 42199081 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:465 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:386
ec02ef8e2017f1eaa8c504020100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
428c9544e939a70845103eaa0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
04dfff753b3fee04fe387ae20100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
8d2615f1c5d5a9fa3237bd020100000001000000 LOCAL_REFERENCE 42199078 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:465 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:386
31c9682b200b2d1b644b61330100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ffffffffffffffffffffffff0100000002000000 LOCAL_REFERENCE 5988152 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:364 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489
7c4cdd595e2d651212e0e0f40100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
542095f40f30330b64b776030100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
6c4236c60c7039a5b01582820100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
cd801d08d3d3200ecb3b96300100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
a979b2ab78b25508d447215b0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
f2e96d4d30ce6c6fe5e886fb0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
1b3bebaeb69b5f015fc9824d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
c809e2d1e37165f3a5feebbd0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
9bbe789bb6461f2495e0a3050100000001000000 LOCAL_REFERENCE 42199075 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
34224adffd956ed835b554790100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
62d9507a9ba07c6a91f0b94d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
e109113d5265e0b6bb3109950100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
97d7f87af5cab5ea7768de4b0100000001000000 LOCAL_REFERENCE ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_train:254 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:continue_training:382 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_runner.py:_process_trial_restore:889
a2a02f12fa4dad6732a8fbfd0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
3fe6746b413f0cab8b505c750100000001000000 LOCAL_REFERENCE 42199075 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
fa9a2eca7a906582e28a37520100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
2dd655a04c4c9e7b31e833d90100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
52ab9ef43b8c725b3a8411e30100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
5e35bc218ff40072ab4b6de60100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ffffffffffffffffffffffff0100000003000000 LOCAL_REFERENCE 15 (put object) | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:create_experiment:365 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune:433 | /usr/local/lib/python3.8/site-packages/lumi/training/tune.py:tune_from_config:489
8f3091729f95c547ea0f08670100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
3b71c2be3ac5768dbd84db510100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
5ad3071645d44b70e55222870100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
eb9d7c6ae4e42bec009191420100000001000000 LOCAL_REFERENCE ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_train:254 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:continue_training:382 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_runner.py:_process_trial_restore:889
fa646152df29b0ec20b536150100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
0a0923cf38bccb7c537912cd0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ffe736debb698303644b61330100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:465 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:386
558d96769a64fbc53237bd020100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
a867f14f359544cb43286c8a0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
d11ff74e3d829171d460e13d0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
fa867a1223971b151536282b0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
9d8b19a68856b3f09d6818d10100000001000000 LOCAL_REFERENCE 42199074 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:_perturb_trial:465 | /usr/local/lib/python3.8/site-packages/ray/tune/schedulers/pbt.py:on_trial_result:386
632264cee885c680e11cf8b30100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ac3cd52ffe01d15eafd485860100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
ffffffffffffffff7768de4b0100000001000000 USED_BY_PENDING_TASK ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_setup_remote_runner:233 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_start_trial:287 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:start_trial:348
01cceb7890ea9b28acff420c0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
5108c0283f4de0ff61f72e510100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
15847c8b9546e919a4bfc3740100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
a24a8dade583182076b20ceb0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
364345483ac2f041c12d198c0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
22f99d3a7af8ae2d3ab7a5270100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
dafc71a4c2ed927e107832830100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
63a5bc0f64190735051376360100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
c94be608d8e39188644b61330100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
f2514873adbe650178e3854c0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
7d6ab7c0719388a6270b5a300100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
cce9e7663d01afd18e72efa60100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
37b0537ddc8cf77f97786a100100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
2db850ebabd1093ef2b843d00100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
832ac0329372d47fb6fb1f330100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
629614ba206cd6f3ba2284750100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
b6ebe15fb136527457ed43020100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
00ab34e4420727ab4491cb510100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
e32c4e4f8da07b24b661618f0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
388edc5c7786a86cbe0c12ed0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
c5b2a9cda63be7b1dcf35dbd0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
a9addba60d77e76fa1b9967c0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
fa0a5b4cf4faf9dd8b505c750100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
430e1692342cd9d45ad5df3c0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
974308e5ae4eb9a40cca1be00100000001000000 LOCAL_REFERENCE 42199078 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
5116ef3c90354faa606093120100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
8aceae866984fb11c8f8705a0100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
e928f0892665dcdc3422fd990100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
6f4c18742f144dbabe14b4460100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
dbe99e63f3c56510bd84db510100000001000000 LOCAL_REFERENCE 42199079 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:save:684 | /usr/local/lib/python3.8/site-packages/ray/tune/trial_executor.py:pause_trial:115 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:pause_trial:393
ffffffffffffffff009191420100000001000000 USED_BY_PENDING_TASK ? (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_setup_remote_runner:233 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_start_trial:287 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:start_trial:348
582860ff7b79d6a38e48ed110100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
2bacfa27b53a75cd73d04bb20100000001000000 LOCAL_REFERENCE 15 (actor call) | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:add:100 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:_stop_trial:326 | /usr/local/lib/python3.8/site-packages/ray/tune/ray_trial_executor.py:stop_trial:371
-----------------------------------------------------------------------------------------------------
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment