Skip to content

Instantly share code, notes, and snippets.

@gbrayut gbrayut/test.sh
Last active Mar 19, 2019

Embed
What would you like to do?
Firewall Testing
#Test all A entries for a DNS record to see if you can connect to port 443:
dig +short microsoft.com. | xargs -I {} nc -v -w2 {} 443
#Example of output
Connection to 191.239.213.197 443 port [tcp/https] succeeded!
Connection to 104.43.195.251 443 port [tcp/https] succeeded!
Connection to 104.40.211.35 443 port [tcp/https] succeeded!
Connection to 23.100.122.175 port 443 (tcp) timed out: Operation now in progress
Connection to 23.96.52.53 443 port [tcp/https] succeeded!
#Test using a string as input (printf prevents issue with \n from echo. echo -n also works)
printf %b '10.1.0.1 10.2.0.2 10.3.0.3 10.4.0.4' | xargs -d ' ' -I {} nc -v -w2 {} 443
echo -n test-{east,west,south,north}-server.example.com | xargs -d ' ' -I {} nc -v -w2 {} 443
#Example of output
nc: connect to 10.1.0.1 443 port [tcp/https] succeeded!
nc: connect to 10.2.0.2 port 443 (tcp) failed: Connection refused
nc: connect to 10.3.0.3 port 443 (tcp) timed out: Operation now in progress
nc: connect to 10.4.0.4 port 443 (tcp) timed out: Operation now in progress
#Run multiple commands. This brace expands to 10.x.0.1, .5, and .9 for x=1 thru 10
echo -n 10.{1..10}.0.{1,5,9} | xargs -d ' ' -I {} bash -c "echo -e '\n\n\nIP: {}'; curl -vsm 3 http://{}/"
#Can also test for local route details (example: see which local interface would be used for egress)
printf %b '192.168.0.4 192.168.76.36 192.168.4.100 10.120.100.1' | xargs -d ' ' -I {} ip route get {}
local 192.168.0.4 dev lo src 192.168.0.4
cache <local>
192.168.76.36 via 192.168.0.1 dev enp0s31f6 src 192.168.0.4
cache
192.168.4.100 dev lxdbr0 src 192.168.4.1
cache <local>
10.120.100.1 dev vpn0 src 172.16.2.1
cache
#Use mtr for traceroute, send output to stdout so you can copy/paste. See also --tcp or --port options if ICMP is blocked
sudo mtr -n -r -c 5 microsoft.com
Start: Thu May 31 12:12:12 2018
HOST: gbmint02 Loss% Snt Last Avg Best Wrst StDev
1.|-- 192.168.0.1 0.0% 5 0.4 0.4 0.4 0.4 0.0
2.|-- 96.120.96.89 0.0% 5 8.1 7.7 7.3 8.1 0.0
3.|-- 68.87.170.221 0.0% 5 7.6 7.7 7.6 7.9 0.0
4.|-- 69.139.231.85 0.0% 5 7.8 7.9 7.6 8.3 0.0
5.|-- 68.86.90.225 0.0% 5 20.0 20.4 19.9 21.1 0.0
6.|-- 68.86.84.226 0.0% 5 20.6 29.6 20.2 64.6 19.6
7.|-- 68.86.83.94 0.0% 5 20.0 19.7 19.0 21.3 0.9
8.|-- 50.242.149.162 0.0% 5 19.0 19.7 19.0 21.1 0.5
9.|-- 104.44.4.247 0.0% 5 138.1 138.0 136.0 140.1 1.4
10.|-- 104.44.4.110 0.0% 5 138.6 137.9 135.6 140.1 1.6
11.|-- 104.44.4.96 0.0% 5 136.2 136.9 136.2 137.8 0.5
12.|-- 104.44.4.175 0.0% 5 135.9 137.4 135.9 140.8 1.9
13.|-- 104.44.4.77 0.0% 5 136.7 136.9 136.0 137.6 0.5
14.|-- 104.44.4.52 0.0% 5 143.8 138.0 135.7 143.8 3.2
15.|-- 104.44.4.65 0.0% 5 137.7 136.8 135.6 137.7 0.0
16.|-- 104.44.5.8 0.0% 5 136.9 136.6 136.1 137.2 0.0
17.|-- 104.44.7.71 0.0% 5 136.1 136.1 135.8 136.4 0.0
18.|-- 104.44.7.39 0.0% 5 135.9 136.3 135.9 136.5 0.0
19.|-- 104.44.11.82 0.0% 5 135.7 135.4 135.2 135.7 0.0
20.|-- ??? 100.0 5 0.0 0.0 0.0 0.0 0.0
#If you use iptables to restrict outbound connections to specific user accounts you can test
sudo -u username nmap -Pn 1.1.1.1 -p53
sudo -u username nc -v -w2 1.1.1.1 53
#use awk to see connections to local port 9090 and show count of remote ips
sudo netstat -nap | awk '/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c
#similar, but only include lines with local ip:80 or :443
sudo netstat -nap | awk "/$(hostname -i):(80|443) / && ! /::1:| LISTEN / {match(\$5, /(.*):(.*)/, m); print m[1]}" |sort|uniq -c
#Watch changes. Also wrap in $'command \'nested string\' |command' for better escaping of only nested single quotes
watch -d -n2 $'sudo netstat -nap | awk \'/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}\' |sort|uniq -c'
#Watch local 80/443, excluding TIME_WAIT (rough estimate of active connections to local webserver). Change m[1] to \$0 to see raw data.
watch -d -n2 "sudo netstat -nap | awk '/$(hostname -i):(80|443) / && ! /::1:| LISTEN | TIME_WAIT/ {match(\$5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c"
#You can do more with gawk (sorting and counting in one pass)
#The commands below assume you are on a system that listens on 80/443 and then fetches responses
#from other systems (using various ports)
#View client IP count by dst port. Limits to tcp, exludes listen state, requires local dst port of 80/443,
#then counts src -> local port and displays any with a count more than 5
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|443)$/ {split($4,dst,":");split($5,src,":");groups[(src[1] "-> :" dst[2])]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {print groups[g],g}}'
378 10.100.0.11-> :443
365 10.100.0.12-> :443
363 10.100.0.13-> :443
358 10.100.0.14-> :443
357 10.100.0.15-> :443
#The above results show 350-380 connections from (in this case) an upstream WAF/LB cluster with 5 nodes
#establishing pooled connections to our local port 443
#View fetches in flight (or recently finished). Limits to tcp, excludes listen state and local port 80/443
#(rough filter for sockets where we are client not server), and again displays counts greater than 5
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}'
Count of fetches in flight
43 10.0.0.101:80
11 10.0.1.101:80
9 10.0.2.101:80
7 10.0.3.101:80
#Same as above but in format for watch or remote salt cmd.run
watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}\''
#This shows the count of connections we have to port 80 on 4 different backend web servers
#Same as above but also break down connection state details
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}'
Count of fetches in flight
43 10.0.0.101:80 TIME_WAIT=27
9 10.0.1.101:80 ESTABLISHED=6 TIME_WAIT=3
8 10.0.2.101:80 ESTABLISHED=6 TIME_WAIT=2
7 10.0.3.101:80 ESTABLISHED=7
#Same as above but in format for watch or remote salt cmd.run
watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}\''
#Here TIME_WAIT represents closed connections waiting for any final packets
#(hard coded at 60s... this is NOT tcp_fin_timeout, see https://stackoverflow.com/a/35000966/17373 )
#You can even see those countdown timers using -o on netstat or ss
# See countdown timer for all TIME_WAIT sockets in 10.0.0.0-10.255.255.255
ss --numeric -o state time-wait dst 10.0.0.0/8
NetidRecv-Q Send-Q Local Address:Port Peer Address:Port
tcp 0 0 192.168.100.1:57516 10.0.0.101:80 timer:(timewait,55sec,0)
tcp 0 0 192.168.100.1:57356 10.0.0.101:80 timer:(timewait,25sec,0)
tcp 0 0 192.168.100.1:57334 10.0.0.101:80 timer:(timewait,6.536ms,0)
tcp 0 0 192.168.100.1:57282 10.0.1.101:80 timer:(timewait,12sec,0)
tcp 0 0 192.168.100.1:57418 10.0.2.101:80 timer:(timewait,38sec,0)
#TIME_WAIT does NOT mean there is any issue with the client or server. Especially if tcp_tw_reuse is enable,
#these sockets will get reused for new connections if there won't be any conflict in TCP Segment numbers.
#The above server with more TIME_WAIT may just be an API server that has a bunch of small request/responces,
#and those sockets don't get reused as much.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.