surki

node:
  id: foo
  cluster: foo_envoy_test

admin:
  access_log_path: "/dev/stdout"
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0

surki

# Start gdb-server in port 50505 using above instructions
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay -s 50505 -k /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# In Emacs, start gdb client and connect to it
(gdb) set sysroot /home/surki/tmp/some_app_crash/rr/amzlinux_rootfs
(gdb) target remote :50505
(gdb) source ~/tmp/some_app_crash/rr/amzlinux_rootfs/gdb_ruby.py
(gdb) continue

surki

# Run this from the directory where amzlinux_rootfs is present
 
# Once: mount the appropriate procfs
for f in /proc /sys /dev; do sudo mount --bind $f ./amzlinux_rootfs/$f; done
 
# Enter the rootfs and replay
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# now we should be in gdb prompt, use gdb commands to run/stop/inspect/watch/reverse-cont etc

surki

exec /usr/local/bin/rr -F record -n \
 --disable-cpuid-features 0x80050440,0x40140400 --disable-cpuid-features-ext 0xffffd854,0xffffffff,0xf3ffffff \
 --disable-cpuid-features-xsave 0xfffffffe \
 "/usr/local/bin/ruby" "$@"

surki

[root@ip-172-16-20-17 ec2-user]# perf stat -e r5101c4 true
 
Performance counter stats for 'true':
 
           58,936      r5101c4
 
      0.000669296 seconds time elapsed

surki

[root@ip-172-16-20-17 ec2-user]# curl http://instance-data/latest/meta-data/instance-type
i3.16xlarge
 
[root@ip-172-16-20-17 ec2-user]# cat /proc/cpuinfo | grep "model name"
model name      : Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz
 
[root@ip-172-16-20-17 ec2-user]# dmesg | grep -i perf
[   22.307535] Performance Events: Broadwell events, core PMU driver.
 

surki

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 17327.17570]
0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
554             rb_alloc_func_t allocator = RCLASS_EXT(klass)->allocator;
 
 
(rr) bt
#0  0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
#1  0x000055a2008cb58f in rb_obj_alloc (klass=94154328666000) at object.c:1815

surki

Setup

Install rr

# Install rr dependencies
sudo yum install git python2 ccache cmake make gcc gcc-c++ gdb libgcc glibc-devel libstdc++-devel zlib-devel python27-pexpect man-pages capnproto

# Compile/install capnproto if no package available
curl -O https://capnproto.org/capnproto-c++-0.6.1.tar.gz
tar zxf capnproto-c++-0.6.1.tar.gz
pushd capnproto-c++-0.6.1

surki

deployments/
├── base
....
....
│   ├── logging ===============> THIS IS BASE logging SETUP
│   │   ├── filebeat
│   │   │   ├── filebeat-config.yml
│   │   │   ├── filebeat-daemonset.yaml
│   │   │   └── kustomization.yaml

surki

A realworld example showing how mixing ephemeral port range with server listen address can lead to port conflicts (due to unintuitive TCP Simulataneous Open (TSO) on same host, see RFC793)

In this example, we will run a "webserver" and a "health check client" in the same machine

1. Run this on Terminal1 (this is "webserver"):

$ while echo -e "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 6\r\n\r\nhello" | \
    socat -t 0.1 TCP-L:33000,reuseaddr STDIN; do echo "Served request"; done