surki

You can get even more detailed information about a socket from kernel's internal socket struct.
For example, we will try to get nginx listening (on port 81) socket's backlog length

NOTE: You may need to install kernel debug info if not already installed
# yum-config-manager --enable "amzn-main-debuginfo" --enable "amzn-updates-debuginfo"
# yum -y install kernel-debuginfo kernel-devel

Or get socket info for listening socket on port 81
# ss -len | grep :81
tcp    LISTEN     0      511       *:81                    *:*                   ino:29842919 sk:55 <->

surki

A realworld example showing how mixing ephemeral port range with server listen address can lead to port conflicts (due to unintuitive TCP Simulataneous Open (TSO) on same host, see RFC793)

In this example, we will run a "webserver" and a "health check client" in the same machine

1. Run this on Terminal1 (this is "webserver"):

$ while echo -e "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 6\r\n\r\nhello" | \
    socat -t 0.1 TCP-L:33000,reuseaddr STDIN; do echo "Served request"; done

surki

deployments/
├── base
....
....
│   ├── logging ===============> THIS IS BASE logging SETUP
│   │   ├── filebeat
│   │   │   ├── filebeat-config.yml
│   │   │   ├── filebeat-daemonset.yaml
│   │   │   └── kustomization.yaml

surki

Setup

Install rr

# Install rr dependencies
sudo yum install git python2 ccache cmake make gcc gcc-c++ gdb libgcc glibc-devel libstdc++-devel zlib-devel python27-pexpect man-pages capnproto

# Compile/install capnproto if no package available
curl -O https://capnproto.org/capnproto-c++-0.6.1.tar.gz
tar zxf capnproto-c++-0.6.1.tar.gz
pushd capnproto-c++-0.6.1

surki

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 17327.17570]
0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
554             rb_alloc_func_t allocator = RCLASS_EXT(klass)->allocator;
 
 
(rr) bt
#0  0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
#1  0x000055a2008cb58f in rb_obj_alloc (klass=94154328666000) at object.c:1815

surki

[root@ip-172-16-20-17 ec2-user]# curl http://instance-data/latest/meta-data/instance-type
i3.16xlarge
 
[root@ip-172-16-20-17 ec2-user]# cat /proc/cpuinfo | grep "model name"
model name      : Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz
 
[root@ip-172-16-20-17 ec2-user]# dmesg | grep -i perf
[   22.307535] Performance Events: Broadwell events, core PMU driver.
 

surki

[root@ip-172-16-20-17 ec2-user]# perf stat -e r5101c4 true
 
Performance counter stats for 'true':
 
           58,936      r5101c4
 
      0.000669296 seconds time elapsed

surki

exec /usr/local/bin/rr -F record -n \
 --disable-cpuid-features 0x80050440,0x40140400 --disable-cpuid-features-ext 0xffffd854,0xffffffff,0xf3ffffff \
 --disable-cpuid-features-xsave 0xfffffffe \
 "/usr/local/bin/ruby" "$@"

surki

# Run this from the directory where amzlinux_rootfs is present
 
# Once: mount the appropriate procfs
for f in /proc /sys /dev; do sudo mount --bind $f ./amzlinux_rootfs/$f; done
 
# Enter the rootfs and replay
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# now we should be in gdb prompt, use gdb commands to run/stop/inspect/watch/reverse-cont etc

surki

# Start gdb-server in port 50505 using above instructions
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay -s 50505 -k /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# In Emacs, start gdb client and connect to it
(gdb) set sysroot /home/surki/tmp/some_app_crash/rr/amzlinux_rootfs
(gdb) target remote :50505
(gdb) source ~/tmp/some_app_crash/rr/amzlinux_rootfs/gdb_ruby.py
(gdb) continue