Firewall Testing
| #Test all A entries for a DNS record to see if you can connect to port 443: | |
| dig +short microsoft.com. | xargs -I {} nc -v -w2 {} 443 | |
| #Example of output | |
| Connection to 191.239.213.197 443 port [tcp/https] succeeded! | |
| Connection to 104.43.195.251 443 port [tcp/https] succeeded! | |
| Connection to 104.40.211.35 443 port [tcp/https] succeeded! | |
| Connection to 23.100.122.175 port 443 (tcp) timed out: Operation now in progress | |
| Connection to 23.96.52.53 443 port [tcp/https] succeeded! | |
| #Test using a string as input (printf prevents issue with \n from echo. echo -n also works) | |
| printf %b '10.1.0.1 10.2.0.2 10.3.0.3 10.4.0.4' | xargs -d ' ' -I {} nc -v -w2 {} 443 | |
| echo -n test-{east,west,south,north}-server.example.com | xargs -d ' ' -I {} nc -v -w2 {} 443 | |
| #Example of output | |
| nc: connect to 10.1.0.1 443 port [tcp/https] succeeded! | |
| nc: connect to 10.2.0.2 port 443 (tcp) failed: Connection refused | |
| nc: connect to 10.3.0.3 port 443 (tcp) timed out: Operation now in progress | |
| nc: connect to 10.4.0.4 port 443 (tcp) timed out: Operation now in progress | |
| #Run multiple commands. This brace expands to 10.x.0.1, .5, and .9 for x=1 thru 10 | |
| echo -n 10.{1..10}.0.{1,5,9} | xargs -d ' ' -I {} bash -c "echo -e '\n\n\nIP: {}'; curl -vsm 3 http://{}/" | |
| #Can also test for local route details (example: see which local interface would be used for egress) | |
| printf %b '192.168.0.4 192.168.76.36 192.168.4.100 10.120.100.1' | xargs -d ' ' -I {} ip route get {} | |
| local 192.168.0.4 dev lo src 192.168.0.4 | |
| cache <local> | |
| 192.168.76.36 via 192.168.0.1 dev enp0s31f6 src 192.168.0.4 | |
| cache | |
| 192.168.4.100 dev lxdbr0 src 192.168.4.1 | |
| cache <local> | |
| 10.120.100.1 dev vpn0 src 172.16.2.1 | |
| cache | |
| #Use mtr for traceroute, send output to stdout so you can copy/paste. See also --tcp or --port options if ICMP is blocked | |
| sudo mtr -n -r -c 5 microsoft.com | |
| Start: Thu May 31 12:12:12 2018 | |
| HOST: gbmint02 Loss% Snt Last Avg Best Wrst StDev | |
| 1.|-- 192.168.0.1 0.0% 5 0.4 0.4 0.4 0.4 0.0 | |
| 2.|-- 96.120.96.89 0.0% 5 8.1 7.7 7.3 8.1 0.0 | |
| 3.|-- 68.87.170.221 0.0% 5 7.6 7.7 7.6 7.9 0.0 | |
| 4.|-- 69.139.231.85 0.0% 5 7.8 7.9 7.6 8.3 0.0 | |
| 5.|-- 68.86.90.225 0.0% 5 20.0 20.4 19.9 21.1 0.0 | |
| 6.|-- 68.86.84.226 0.0% 5 20.6 29.6 20.2 64.6 19.6 | |
| 7.|-- 68.86.83.94 0.0% 5 20.0 19.7 19.0 21.3 0.9 | |
| 8.|-- 50.242.149.162 0.0% 5 19.0 19.7 19.0 21.1 0.5 | |
| 9.|-- 104.44.4.247 0.0% 5 138.1 138.0 136.0 140.1 1.4 | |
| 10.|-- 104.44.4.110 0.0% 5 138.6 137.9 135.6 140.1 1.6 | |
| 11.|-- 104.44.4.96 0.0% 5 136.2 136.9 136.2 137.8 0.5 | |
| 12.|-- 104.44.4.175 0.0% 5 135.9 137.4 135.9 140.8 1.9 | |
| 13.|-- 104.44.4.77 0.0% 5 136.7 136.9 136.0 137.6 0.5 | |
| 14.|-- 104.44.4.52 0.0% 5 143.8 138.0 135.7 143.8 3.2 | |
| 15.|-- 104.44.4.65 0.0% 5 137.7 136.8 135.6 137.7 0.0 | |
| 16.|-- 104.44.5.8 0.0% 5 136.9 136.6 136.1 137.2 0.0 | |
| 17.|-- 104.44.7.71 0.0% 5 136.1 136.1 135.8 136.4 0.0 | |
| 18.|-- 104.44.7.39 0.0% 5 135.9 136.3 135.9 136.5 0.0 | |
| 19.|-- 104.44.11.82 0.0% 5 135.7 135.4 135.2 135.7 0.0 | |
| 20.|-- ??? 100.0 5 0.0 0.0 0.0 0.0 0.0 | |
| #If you use iptables to restrict outbound connections to specific user accounts you can test | |
| sudo -u username nmap -Pn 1.1.1.1 -p53 | |
| sudo -u username nc -v -w2 1.1.1.1 53 |
| #use awk to see connections to local port 9090 and show count of remote ips | |
| sudo netstat -nap | awk '/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c | |
| #similar, but only include lines with local ip:80 or :443 | |
| sudo netstat -nap | awk "/$(hostname -i):(80|443) / && ! /::1:| LISTEN / {match(\$5, /(.*):(.*)/, m); print m[1]}" |sort|uniq -c | |
| #Watch changes. Also wrap in $'command \'nested string\' |command' for better escaping of only nested single quotes | |
| watch -d -n2 $'sudo netstat -nap | awk \'/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}\' |sort|uniq -c' | |
| #Watch local 80/443, excluding TIME_WAIT (rough estimate of active connections to local webserver). Change m[1] to \$0 to see raw data. | |
| watch -d -n2 "sudo netstat -nap | awk '/$(hostname -i):(80|443) / && ! /::1:| LISTEN | TIME_WAIT/ {match(\$5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c" | |
| #You can do more with gawk (sorting and counting in one pass) | |
| #The commands below assume you are on a system that listens on 80/443 and then fetches responses | |
| #from other systems (using various ports) | |
| #View client IP count by dst port. Limits to tcp, exludes listen state, requires local dst port of 80/443, | |
| #then counts src -> local port and displays any with a count more than 5 along with socket counts | |
| uptime;netstat -na > /dev/shm/netstat; echo Total Established $(grep "[^.][^1]:4444.*ESTABLISHED" /dev/shm/netstat | wc -l); gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|443)$/ {split($4,dst,":");split($5,src,":");groups[(src[1] "-> :" dst[2])]++;states[(src[1] "-> :" dst[2])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat | |
| 13:05:09 up 1 day, 5:13, 1 user, load average: 1.06, 0.66, 0.66 | |
| Total Established 106 | |
| 378 10.100.0.11-> :443 ESTABLISHED=300 TIME_WAIT=78 | |
| 365 10.100.0.12-> :443 TIME_WAIT=250 ESTABLISHED=115 | |
| 363 10.100.0.13-> :443 ESTABLISHED=363 | |
| 358 10.100.0.14-> :443 TIME_WAIT=358 | |
| 357 10.100.0.15-> :443 ESTABLISHED=200 TIME_WAIT=157 | |
| #The above results show 350-380 connections from (in this case) an upstream WAF/LB cluster with 5 nodes | |
| #establishing pooled connections to our local port 443 | |
| # Another for loop to monitor a go server in a container (fixes issue with ipv6 source addresses): | |
| for run in `seq 1 10`; do uptime;netstat -nat > /dev/shm/netstat; echo "Totals: $(curl -s 127.0.0.1:80/metrics|grep '^go_goroutines') Established $(grep 'ESTABLISHED' /dev/shm/netstat | wc -l)"; gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|3000|3001)$/{ndst=split($4,dst,":");nsrc=split($5,src,":");groups[(src[nsrc-1] "-> :" dst[ndst])]++;states[(src[nsrc-1] "-> :" dst[ndst])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat; sleep 600;done; | |
| #View fetches in flight (or recently finished). Limits to tcp, excludes listen state and local port 80/443 | |
| #(rough filter for sockets where we are client not server), and again displays counts greater than 5 | |
| netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}' | |
| Count of fetches in flight | |
| 43 10.0.0.101:80 | |
| 11 10.0.1.101:80 | |
| 9 10.0.2.101:80 | |
| 7 10.0.3.101:80 | |
| #Same as above but in format for watch or remote salt cmd.run | |
| watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}\'' | |
| #This shows the count of connections we have to port 80 on 4 different backend web servers | |
| #Same as above but also break down connection state details | |
| netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' | |
| Count of fetches in flight | |
| 43 10.0.0.101:80 TIME_WAIT=27 | |
| 9 10.0.1.101:80 ESTABLISHED=6 TIME_WAIT=3 | |
| 8 10.0.2.101:80 ESTABLISHED=6 TIME_WAIT=2 | |
| 7 10.0.3.101:80 ESTABLISHED=7 | |
| #Same as above but in format for watch or remote salt cmd.run | |
| watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}\'' | |
| #Here TIME_WAIT represents closed connections waiting for any final packets | |
| #(hard coded at 60s... this is NOT tcp_fin_timeout, see https://stackoverflow.com/a/35000966/17373 ) | |
| #You can even see those countdown timers using -o on netstat or ss | |
| # See countdown timer for all TIME_WAIT sockets in 10.0.0.0-10.255.255.255 | |
| ss --numeric -o state time-wait dst 10.0.0.0/8 | |
| NetidRecv-Q Send-Q Local Address:Port Peer Address:Port | |
| tcp 0 0 192.168.100.1:57516 10.0.0.101:80 timer:(timewait,55sec,0) | |
| tcp 0 0 192.168.100.1:57356 10.0.0.101:80 timer:(timewait,25sec,0) | |
| tcp 0 0 192.168.100.1:57334 10.0.0.101:80 timer:(timewait,6.536ms,0) | |
| tcp 0 0 192.168.100.1:57282 10.0.1.101:80 timer:(timewait,12sec,0) | |
| tcp 0 0 192.168.100.1:57418 10.0.2.101:80 timer:(timewait,38sec,0) | |
| #TIME_WAIT does NOT mean there is any issue with the client or server. Especially if tcp_tw_reuse is enable, | |
| #these sockets will get reused for new connections if there won't be any conflict in TCP Segment numbers. | |
| #The above server with more TIME_WAIT may just be an API server that has a bunch of small request/responces, | |
| #and those sockets don't get reused as much. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment