# While top is running, you can press c to toggle between showing the process name and the command line. To remember the toggle state for next time, press W to save the current configuration to ~/.toprc.
~ $ rbtrace -p 39 -e 'Thread.list'
*** run `sudo sysctl kernel.msgmnb=1048576` to prevent losing events (currently: 16384 bytes)
*** attached to process 39
>> Thread.list
=> [#<Thread:0x000055d0b4072d78 sleep_forever>, #<Thread:0x000055d0bf3d50f0 /app/vendor/bundle/ruby/2.7.0/gems/activerecord-6.0.4.8/lib/active_record/connection_adapters/abstract/connection_pool.rb:334 sleep>, #<Thread:0x000055d0c4db3e78 eval:6 run>]
*** detached from process 39~ $ rbtrace -p 79 -e 'Thread.list'
*** run `sudo sysctl kernel.msgmnb=1048576` to prevent losing events (currently: 16384 bytes)
*** attached to process 79
>> Thread.list
=> [#<Thread:0x000055d0b4072d78 sleep_forever>, #<Thread:0x000055d0c4df31b8@puma wrkr check /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/cluster/worker.rb:36 sleep>, #<Thread:0x000055d0c4df2b28@puma srv tp 001 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>, #<Thread:0x000055d0c4df2920@puma srv tp 002 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c4df26c8@puma srv tp 003 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c4df2448@puma srv tp 004 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c4df20d8@puma srv tp 005 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c4df1b38@puma srv tp 006 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c4df1868@puma srv tp 007 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c4df1570@puma srv tp 008 /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:104 sleep_forever>,
#<Thread:0x000055d0c74cd798@puma reactor /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/reactor.rb:37 sleep>,
#<Thread:0x000055d0c74cd658@puma srv threadpool reaper /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:307 sleep>,
#<Thread:0x000055d0c74cd518@puma srv threadpool trimmer /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/thread_pool.rb:307 sleep>,
#<Thread:0x000055d0c74cd388@puma srv /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/server.rb:259 sleep>,
#<Thread:0x000055d0c74cd298@puma stat pld /app/vendor/bundle/ruby/2.7.0/gems/puma-5.6.4/lib/puma/cluster/worker.rb:116 sleep>,
#<Thread:0x000055d0c4de4550 eval:6 run>]
*** detached from process 79~ $ ps -p $PID -o pid,vsz=MEMORY -o user,group=GROUP -o comm,args=ARGS
PID MEMORY USER GROUP COMMAND ARGS
79 529560 u10331 dyno ruby puma: cluster worker 0: 39 [app]
~ $ ps -w -p $PID -o pid,vsz=MEMORY -o user,group=GROUP -o comm,args=ARGS
PID MEMORY USER GROUP COMMAND ARGS
39 502780 u10331 dyno ruby puma 5.6.4 (unix:/tmp/nginx.socket) [app]rbtrace -b
rbtrace -h
~ $ rbtrace -p 39 -h
*** run `sudo sysctl kernel.msgmnb=1048576` to prevent losing events (currently: 16384 bytes)
*** attached to process 39
*** timed out waiting for eval response
Heapdump being written to /tmp/dump20220622-279-822l8s
*** detached from process 39
~ $ ls -lh /tmp/dump20220622-279-822l8s
/tmp $ ruby diff.rb dump20220622-289-rhzhr1 dump20220622-295-o0ivu2 dump20220622-408-1jo7nuv
Leaked 2334 STRING objects of size 57991/150358 at: :
Leaked 393 DATA objects of size 0/1083040 at: :
Leaked 353 IMEMO objects of size 0/15088 at: :
Leaked 211 ARRAY objects of size 0/10440 at: :
Leaked 53 HASH objects of size 0/8928 at: :
Leaked 50 MATCH objects of size 0/10000 at: :
Leaked 1 FILE objects of size 0/232 at: :
# no cluster mode, single process
/tmp $ ruby diff.rb dump20220622-108-y2u34b dump20220622-127-676bmw dump20220622-136-1tke0l1
Leaked 44844 STRING objects of size 13775695/15879382 at: :
Leaked 9357 IMEMO objects of size 0/599012 at: :
Leaked 7801 HASH objects of size 0/1245624 at: :
Leaked 7631 ARRAY objects of size 0/493440 at: :
Leaked 5187 OBJECT objects of size 0/532752 at: :
Leaked 1510 DATA objects of size 0/1307049 at: :
Leaked 1259 MATCH objects of size 0/211752 at: :
Leaked 273 STRUCT objects of size 0/16168 at: :
Leaked 173 CLASS objects of size 0/93096 at: :
Leaked 148 SYMBOL objects of size 5033/5920 at: :
Leaked 99 BIGNUM objects of size 0/3960 at: :
Leaked 72 ICLASS objects of size 0/2880 at: :
Leaked 42 REGEXP objects of size 0/23413 at: :
Leaked 25 FILE objects of size 0/22216 at: :
Leaked 10 ZOMBIE objects of size 0/0 at: :
Leaked 1 MODULE objects of size 0/552 at: :
bundle exec rbtrace -p 40 -ihttps://stackify.com/how-does-ruby-garbage-collection-work-a-simple-tutorial/
- RUBY_GC_HEAP_INIT_SLOTS: Initial allocation slots.
- RUBY_GC_HEAP_FREE_SLOTS: Prepare at least this amount of slots after GC. Allocate slots if there aren’t enough slots.
- RUBY_GC_HEAP_GROWTH_FACTOR: Allocate slots by this factor. (next slots number) = (current slots number) * (this factor)
- RUBY_GC_HEAP_GROWTH_MAX_SLOTS: The allocation rate is limited to this number of slots.
- RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO: Allocate additional pages when the number of free slots is lower than the value (total_slots * (this ratio)).
- RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO: Allocate slots to satisfy this formula: free_slots = total_slots * goal_ratio. In other words, prepare (total_slots * goal_ratio) free slots. If this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
- RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO: Allow to free pages when the number of free slots is greater than the value (total_slots * (this ratio)).
- RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR: Do full GC when the number of old objects is more than R * N where R is this factor and N is the number of old objects just after last full GC.
GC.start is another good Ruby garbage collection command to know. It kicks off a garbage collection cycle. If you have a long-running script that’s chewing up memory, you can have it call GC.start periodically to keep the memory usage under control.
check threads in progress
$ top -H -p 48
~ $ ps -T -p 48 -cww
PID SPID CLS PRI TTY TIME CMD
48 48 TS 19 ? 00:00:14 ruby
48 86 TS 19 ? 00:00:00 connection_poo*
48 97 TS 19 ? 00:00:03 puma srv tp 001
48 98 TS 19 ? 00:00:03 puma srv tp 002
48 99 TS 19 ? 00:00:03 puma srv tp 003
48 100 TS 19 ? 00:00:03 puma srv tp 004
48 101 TS 19 ? 00:00:03 puma srv tp 005
48 102 TS 19 ? 00:00:02 puma srv tp 006
48 103 TS 19 ? 00:00:03 puma srv tp 007
48 104 TS 19 ? 00:00:02 puma srv tp 008
48 105 TS 19 ? 00:00:10 puma srv tp 009
48 106 TS 19 ? 00:00:03 puma srv tp 010
48 107 TS 19 ? 00:00:03 puma srv tp 011
48 108 TS 19 ? 00:00:03 puma srv tp 012
48 109 TS 19 ? 00:00:03 puma srv tp 013
48 110 TS 19 ? 00:00:04 puma srv tp 014
48 111 TS 19 ? 00:00:03 puma srv tp 015
48 112 TS 19 ? 00:00:03 puma srv tp 016
48 113 TS 19 ? 00:00:03 puma srv tp 017
48 114 TS 19 ? 00:00:03 puma srv tp 018
48 115 TS 19 ? 00:00:03 puma srv tp 019
48 116 TS 19 ? 00:00:03 puma srv tp 020
48 117 TS 19 ? 00:00:00 puma reactor
48 118 TS 19 ? 00:00:00 thread_pool.rb*
48 119 TS 19 ? 00:00:00 thread_pool.rb*
48 120 TS 19 ? 00:00:00 puma srv
48 121 TS 19 ? 00:00:01 agent_thread.r*
48 131 TS 19 ? 00:00:00 reporter.rb:30
48 134 TS 19 ? 00:00:00 session_flushe*
48 135 TS 19 ? 00:00:00
48 141 TS 19 ? 00:00:01 io-worker-8
48 146 TS 19 ? 00:00:01 io-worker-13
48 147 TS 19 ? 00:00:01 io-worker-14
48 227 TS 19 ? 00:00:00 worker-1
48 1051 TS 19 ? 00:00:00 io-worker-20
48 1141 TS 19 ? 00:00:00 io-worker-23
48 1142 TS 19 ? 00:00:00 io-worker-24
48 1147 TS 19 ? 00:00:00 io-worker-25
48 1148 TS 19 ? 00:00:00 io-worker-22
48 1285 TS 19 ? 00:00:00 io-worker-28
48 1351 TS 19 ? 00:00:00 io-worker-29
48 1390 TS 19 ? 00:00:00
48 1391 TS 19 ? 00:00:00
48 1392 TS 19 ? 00:00:00
48 1393 TS 19 ? 00:00:00
48 1394 TS 19 ? 00:00:00
48 1395 TS 19 ? 00:00:00
48 1396 TS 19 ? 00:00:00
48 1397 TS 19 ? 00:00:00Append 2022-06-27 3:19PM set WEB_CONCURRENCY to 0 2022-06-27 5:46 PM set RAILS_MAX_THREADS to 10 2022-06-27 6:03 PM set RAILS_MAX_THREADS to 15 2022-06-27 6:10 PM set RAILS_MAX_THREADS to 20 2022-06-29 11:20 AM set RAILS_MAX_THREADS to 16 2022-06-29 11:20 AM set JEMALLOC_ENABLED to true 2022-06-29 5:19 PM set JEMALLOC_VERSION to 16 2022-06-29 6:20 AM set Upscale to 400
~ $ rbtrace p 40 -h
*** run `sudo sysctl kernel.msgmnb=1048576` to prevent losing events (currently: 16384 bytes)
Traceback (most recent call last):
3: from /app/vendor/bundle/bin/rbtrace:29:in `<main>'
2: from /app/vendor/bundle/bin/rbtrace:29:in `load'
1: from /app/vendor/bundle/ruby/2.7.0/gems/rbtrace-0.4.14/bin/rbtrace:5:in `<top (required)>'
/app/vendor/bundle/ruby/2.7.0/gems/rbtrace-0.4.14/lib/rbtrace/cli.rb:373:in `run': undefined method `size' for nil:NilClass (NoMethodError)rbtrace -p 40 -e 'Thread.new{ require "stackprof"; StackProf.start(mode: :cpu); sleep 2; StackProf.stop; StackProf.results("/tmp/perf"); }'
rbtrace -p 40 -e 'Thread.new{ require "stackprof"; StackProf.start(mode: :cpu); sleep 2; StackProf.stop; StackProf.results("/tmp/perf-2"); }'
rbtrace -p 40 -e 'Thread.new{ require "stackprof"; StackProf.start(mode: :cpu); sleep 2; StackProf.stop; StackProf.results("/tmp/perf-3"); }'
rbtrace -p 40 -e 'Thread.new{require "objspace"; ObjectSpace.trace_object_allocations_start; GC.start(); ObjectSpace.dump_all(output: File.open("/tmp/heap1.json", "w"))}.join'
rbtrace -p 40 -e 'Thread.new{require "objspace"; ObjectSpace.trace_object_allocations_start; GC.start(); ObjectSpace.dump_all(output: File.open("/tmp/heap2.json", "w"))}.join'
rbtrace -p 40 -e 'Thread.new{require "objspace"; ObjectSpace.trace_object_allocations_start; GC.start(); ObjectSpace.dump_all(output: File.open("/tmp/heap3.json", "w"))}.join'
-e 'Thread.new{ require "stackprof"; StackProf.start(mode: :cpu); sleep 2; StackProf.stop; StackProf.results("/tmp/perf-1"); }'
heapy diff heap1.json heap2.json heap3.json > result.txt
$ bundle exec rbtrace -p <SERVER PID HERE> -e 'Thread.new{GC.start;require "objspace";io=File.open("/tmp/ruby-heap.dump", "w"); ObjectSpace.dump_all(output: io); io.close}'
$ rbtrace -p 12345 -m digest
$ rbtrace -p 12345 -h
$ rbtrace -p 40 -e 'ObjectSpace.memsize_of Prometheus::Client.config.data_store'