solana-packet-monitor.sh

#!/bin/bash

# todo: port reverse remapping to nemonic; recognize --dynamic-port-range?;

# run like this: curl -s https://gist.githubusercontent.com/ryoqun/9ad581cdb0d131f12b528faf00eb9652/raw/0000000000..../solana-packet-monitor.sh | nice -n 20 chrt --idle 0 ionice -c 3 bash

set -e
(

if [[ -e /tmp/solana-packet-monitoring.pid ]];
then
  echo "already running? remove /tmp/solana-packet-monitoring.pid"
  exit 1
fi

echo "$(date --utc):$$: started" >> /dev/stderr
type -a nice chrt ionice solana conntrack dig influx || (echo "there was missing binary" && false)

echo $$ > /tmp/solana-packet-monitoring.pid
touch /tmp/solana-packet-monitoring-gossip-nodes /tmp/solana-packet-monitoring-conntrack

echo this will run until you remove /tmp/solana-packet-monitoring.pid...
echo also, all applied system configuration will be reset after OS restart

set -x
private_ip="$(ip route get 8.8.8.8 | head -n1 | awk '{print $7}')"
if [[ $public_ip = "" ]];
then
  public_ip="$(dig -4 TXT +short o-o.myaddr.l.google.com @ns1.google.com | tr -d \")"
fi
host_id="$(solana -ut gossip | grep -F .  | grep -F "$public_ip" | awk '{print $3}')"
if [[ $private_ip = "" || $public_ip = "" || $host_id = "" ]];
then
  echo bad initialization
  exit 1
fi
set +x

## system setup (must be idempotent for repeated multiple invocations)
(
  #set -x
  #sudo apt install conntrack dnsutils influxdb-client
  sudo modprobe nf_conntrack tstamp=1 acct=1
  echo 1 | sudo tee /proc/sys/net/netfilter/nf_conntrack_timestamp
  echo 1 | sudo tee /proc/sys/net/netfilter/nf_conntrack_acct
  sudo iptables -w -t mangle -C PREROUTING -p udp -m conntrack --ctstate NEW -m comment --comment "sol-packet-monitor" ||
    sudo iptables -w -t mangle -I PREROUTING 1 -p udp -m conntrack --ctstate NEW -m comment --comment "sol-packet-monitor"
  # todo: reduce this to avoid overflow in the linux kernel's conntrack table....
  echo $((3600 * 24 * 30)) | sudo tee /proc/sys/net/netfilter/nf_conntrack_udp_timeout
  echo $((3600 * 24 * 30)) | sudo tee /proc/sys/net/netfilter/nf_conntrack_udp_timeout_stream
  sudo conntrack -F 2> /dev/null ## this is needed to wipe all possibly pre-existing conntrack entries with acct=0
)

IPTABLES_SLEEP=${IPTABLES_SLEEP:-1}

function influx_with_retry() {
  for i in {1..10}
  do
    if influx "$@"
    then
      return 0
    else
      echo "$(date --utc): retrying influx ($i) $@..." >> /dev/stderr      
      sleep 1
    fi
  done
  # greatly reduce activity but still don't terminate the monitoring altogether...
  sleep 600
  return 0
}

# to address short data retension, always add entry.
#influx_with_retry -host metrics.solana.com -ssl -username ro -password topsecret -database tds -execute \
#  'select * from conntrack_enabled_test' |
#  tail -n +4 |
#  grep -q "$host_id" ||
influx_with_retry -host metrics.solana.com -ssl -username testnet_write -password c4fa841aa918bf8274e3e2a44d77568d9861b3ea -database tds -execute \
  "insert conntrack_enabled_test,host_id=$host_id public_ip=\"$public_ip\""

(#set -x &&
    sudo ss --listen --process --udp| 
      grep solana-validato | 
      awk '{print $4}' | 
      sort | 
      uniq | 
      sed 's/:/ /g' | 
      awk '{print $2}' | 
      (while read port;
        do
          (
            sudo iptables -w -C INPUT -p udp --dport $port -m comment --comment "sol-packet-monitor" 2> /dev/null ||
            sudo iptables -w -I INPUT 1 -p udp --dport $port -m comment --comment "sol-packet-monitor"
          ) && (
            sudo iptables -w -C OUTPUT -p udp --sport $port -m comment --comment "sol-packet-monitor" 2> /dev/null ||
            sudo iptables -w -I OUTPUT 1 -p udp --sport $port -m comment --comment "sol-packet-monitor"
          )
        done
      ) &&
      sudo iptables -w -t filter -L -nvx --zero &> /dev/null &&
        while [[ -e /tmp/solana-packet-monitoring.pid && $$ = "$(cat /tmp/solana-packet-monitoring.pid)" ]]
        do
	  sudo iptables -w --wait-interval 50000 -t filter -L -nvx --zero > /tmp/solana-packet-monitoring-iptables &&
	  now="$(date --utc '+%s%N')" &&
          measurement=$(
            echo $(
              cat /tmp/solana-packet-monitoring-iptables |
	      grep -vE '^ +0 +0 +' |
              grep -E '(dpt|spt).*sol-packet-monitor' |
              sed -r 's/dpt:([0-9]*)/l\1i/' |
              sed -r 's/spt:([0-9]*)/l\1o/' |
              (while read packets bytes _ _ _ _ _ _ _ port _; do port=${port/:/_} &&
              echo ${port}_packets=$packets ${port}_bytes=$bytes ; done))) &&
          measurement=${measurement// /,} &&
          $_echo influx_with_retry -host metrics.solana.com -ssl -database tds -username testnet_write -password c4fa841aa918bf8274e3e2a44d77568d9861b3ea -execute "insert udp_lport_test,local_host_id=$host_id $measurement $now";
          sleep "$IPTABLES_SLEEP"
        done) &

(#set -x &&
  while [[ -e /tmp/solana-packet-monitoring.pid && $$ = "$(cat /tmp/solana-packet-monitoring.pid)" ]]
  do
    date --utc '+%Yy%mm%dd%Hh%Mm%Ss' |
    grep -q 2021y..m..d..h..m.0s && (
      (
        mv /tmp/solana-packet-monitoring-conntrack{,.bak} &&
        sudo conntrack -L 2> /dev/null > /tmp/solana-packet-monitoring-conntrack &&
        now="$(date --utc '+%s%N')" &&
        PATTERN="$(
          echo $(
            influx_with_retry -host metrics.solana.com -ssl -username ro -password topsecret -database tds -execute \
              'select distinct(public_ip) from conntrack_enabled_test' |
            tail -n +4 |
            grep -v "$public_ip" |
            awk '{print $2}') | tr \  \| )";
        [[ $PATTERN = "" ]] && PATTERN="should_not_match"
        cat /tmp/solana-packet-monitoring-conntrack |
         grep -F "$private_ip" |
         grep -vE "^udp +[0-9]+ +[0-9]+ +src=$private_ip" |
         grep -E "^udp.*($PATTERN)" |
         sed 's/src=/\nsrc=/g' |
         grep ^src= |
         (
          while read _ _ remote_port1 local_port1 packets1 bytes1 _ && read local_ip remote_ip local_port2 remote_port2 packets2 bytes2 _
          do
            echo udp_lport_rport_test,local_host_id=${local_ip/src=},remote_host_id=${remote_ip/dst=} l${local_port1/dport=/}r${remote_port1/sport=/}i ${bytes1/bytes=} l${local_port2/sport=/}r${remote_port2/dport=/}o ${bytes2/bytes=} $now
          done
         ) &&
          cat /tmp/solana-packet-monitoring-conntrack |
            grep -E "^udp +[0-9]+ +[0-9]+ +src=$private_ip" |
            grep -E "^udp.*($PATTERN)" |
            sed 's/src=/\nsrc=/g' |
            grep ^src= | (
              while read local_ip remote_ip local_port1 remote_port1 packets1 bytes1 _ && read _ _ remote_port2 local_port2 packets2 bytes2 _
              do
                echo udp_lport_rport_test,local_host_id=${local_ip/src=},remote_host_id=${remote_ip/dst=} l${local_port2/dport=/}r${remote_port2/sport=/}i ${bytes2/bytes=} l${local_port1/sport=/}r${remote_port1/dport=/}o ${bytes1/bytes=} $now
              done
          )) |
          sed "s/$private_ip/$public_ip/g" |
          grep -vE "($public_ip.*){2}" |
          (false || sed $(
	    (cat /tmp/solana-packet-monitoring-gossip-nodes && while ! solana -ut gossip
	    do
	      echo "$(date --utc): retrying solana gossip..." >> /dev/stderr
	      sleep 1
	    done) | tail -n 20000 | tee /tmp/solana-packet-monitoring-gossip-nodes | grep -F . | sed 's/[.]/[.]/g' | sort | uniq |
            (while read ip _ host_id _; do printf -- "-e %s\n" s/$ip/$host_id/g; done))) &&
           sleep 1);
          sleep "0.01"
        done) |
        ( #set -ux 
          declare -A last_stats;
	  while read -r label key1 total_value1 key2 total_value2 now
          do
	    if [[ $total_value1 != ${total_value1/packets=} || $total_value2 != ${total_value2/packets=} ]]
	    then
	      echo "$(date --utc): bad format: $label $key1 $total_value1 $key2 $total_value2" >> /dev/stderr
	      cp /tmp/solana-packet-monitoring-conntrack{,.bad}
	      #rm /tmp/solana-packet-monitoring.pid
	      #exit 1
	    else
              full_key1="${label}_$key1"
              full_key2="${label}_$key2"
              last_value1=${last_stats[$full_key1]:-$total_value1}
              last_value2=${last_stats[$full_key2]:-$total_value2}
              value1=$(($total_value1 - $last_value1))
              value2=$(($total_value2 - $last_value2))
              last_stats[$full_key1]=$total_value1
              last_stats[$full_key2]=$total_value2
	      if [[ $value1 != "0" || $value2 != "0" ]]
	      then
                $_echo influx_with_retry -host metrics.solana.com -ssl -database tds -username testnet_write -password c4fa841aa918bf8274e3e2a44d77568d9861b3ea -execute "insert $label ${key1}_bytes=$value1,${key2}_bytes=$value2,i_total_bytes=$value1,o_total_bytes=$value2 $now"
	      fi
	    fi
          done)
          
wait

echo "$(date --utc):$$: finished" >> /dev/stderr

) 2>> /tmp/solana-packet-monitoring.stderr.log