Debug recipes for Linux

debug_recipes.txt

# Quick debug network traffic between nginx and php-fpm
# Got from:
# http://systembash.com/content/simple-sysadmin-trick-using-tcpdump-to-sniff-web-server-traffic/

# Just a flow of everything looking like a string
tcpdump -nl -w - -i eth0 -c 500 port 9090 | strings
# Request heads
sudo tcpdump -nl -w - -i eth0 -c 500 port 9090 | strings | grep -E -A5 "^(GET|PUT|POST) "

# strace processes/sub-processes 
#
# For e.g. nodejs script was throwing
#  Error: spawn ENOENT
#    at errnoException (child_process.js:1001:11)
#    at Process.ChildProcess._handle.onexit (child_process.js:792:34)
# Tool did not have any verbose/debug mode.. strace showed that I did
# not have java installed in that machine:
#  9713  execve("/usr/bin/java", ["java", "-jar", "/vagrant/node_modules/protractor"..., "-port", "4444", 
# " -Dwebdriver.chrome.driver=/vagra"...], [/* 30 vars */]) = -1 ENOENT (No such file or directory)

strace -ostrace.txt -f -e trace=process some_script_or_command

# gstack - display call stack
# When process is blocked by I/O and strace doesn't say too much gstack can help identify where exactly it is stuck.
# 
# For e.g. here's a sample call stack from process which is waiting for new messages in gearman queue:
# 
# #0  0x00000036626cc06f in poll () from /lib64/libc.so.6
# #1  0x00007ff0b67f6ac1 in gearman_wait(gearman_universal_st&) () from /usr/local/lib/libgearman.so.8
# #2  0x00007ff0b67f8757 in gearman_worker_grab_job () from /usr/local/lib/libgearman.so.8
# #3  0x00007ff0b67f8a34 in gearman_worker_work () from /usr/local/lib/libgearman.so.8
# #4  0x00007ff0b6a18c9a in zif_gearman_worker_work () from /usr/lib64/php/modules/gearman.so
# # ............ <skipped new relic specific stuff> ..................
# #23 0x0000000000633fb5 in zend_execute_scripts ()
# #24 0x00000000005e36d8 in php_execute_script ()
# #25 0x00000000006be376 in ?? ()
# #26 0x000000366261d9f4 in __libc_start_main () from /lib64/libc.so.6
# #27 0x00000000004226e9 in _start ()
#
# So in this case we can see that it's waiting for data to be received via socket.

gstack 2692

netstat -p | grep 30496