Instantly share code, notes, and snippets.

@gbrayut /test.sh
Last active Jan 17, 2019

Embed
What would you like to do?
Firewall Testing
#Test all A entries for a DNS record to see if you can connect to port 443:
dig +short microsoft.com. | xargs -I {} nc -v -w2 {} 443
#Example of output
Connection to 191.239.213.197 443 port [tcp/https] succeeded!
Connection to 104.43.195.251 443 port [tcp/https] succeeded!
Connection to 104.40.211.35 443 port [tcp/https] succeeded!
Connection to 23.100.122.175 port 443 (tcp) timed out: Operation now in progress
Connection to 23.96.52.53 443 port [tcp/https] succeeded!
#Test using a string as input (printf prevents issue with \n from echo. echo -n also works)
printf %b '10.1.0.1 10.2.0.2 10.3.0.3 10.4.0.4' | xargs -d ' ' -I {} nc -v -w2 {} 443
echo -n test-{east,west,south,north}-server.example.com | xargs -d ' ' -I {} nc -v -w2 {} 443
#Example of output
nc: connect to 10.1.0.1 443 port [tcp/https] succeeded!
nc: connect to 10.2.0.2 port 443 (tcp) failed: Connection refused
nc: connect to 10.3.0.3 port 443 (tcp) timed out: Operation now in progress
nc: connect to 10.4.0.4 port 443 (tcp) timed out: Operation now in progress
#Run multiple commands. This brace expands to 10.x.0.1, .5, and .9 for x=1 thru 10
echo -n 10.{1..10}.0.{1,5,9} | xargs -d ' ' -I {} bash -c "echo -e '\n\n\nIP: {}'; curl -vsm 3 http://{}/"
#Can also test for local route details (example: see which local interface would be used for egress)
printf %b '192.168.0.4 192.168.76.36 192.168.4.100 10.120.100.1' | xargs -d ' ' -I {} ip route get {}
local 192.168.0.4 dev lo src 192.168.0.4
cache <local>
192.168.76.36 via 192.168.0.1 dev enp0s31f6 src 192.168.0.4
cache
192.168.4.100 dev lxdbr0 src 192.168.4.1
cache <local>
10.120.100.1 dev vpn0 src 172.16.2.1
cache
#Use mtr for traceroute, send output to stdout so you can copy/paste. See also --tcp or --port options if ICMP is blocked
sudo mtr -n -r -c 5 microsoft.com
Start: Thu May 31 12:12:12 2018
HOST: gbmint02 Loss% Snt Last Avg Best Wrst StDev
1.|-- 192.168.0.1 0.0% 5 0.4 0.4 0.4 0.4 0.0
2.|-- 96.120.96.89 0.0% 5 8.1 7.7 7.3 8.1 0.0
3.|-- 68.87.170.221 0.0% 5 7.6 7.7 7.6 7.9 0.0
4.|-- 69.139.231.85 0.0% 5 7.8 7.9 7.6 8.3 0.0
5.|-- 68.86.90.225 0.0% 5 20.0 20.4 19.9 21.1 0.0
6.|-- 68.86.84.226 0.0% 5 20.6 29.6 20.2 64.6 19.6
7.|-- 68.86.83.94 0.0% 5 20.0 19.7 19.0 21.3 0.9
8.|-- 50.242.149.162 0.0% 5 19.0 19.7 19.0 21.1 0.5
9.|-- 104.44.4.247 0.0% 5 138.1 138.0 136.0 140.1 1.4
10.|-- 104.44.4.110 0.0% 5 138.6 137.9 135.6 140.1 1.6
11.|-- 104.44.4.96 0.0% 5 136.2 136.9 136.2 137.8 0.5
12.|-- 104.44.4.175 0.0% 5 135.9 137.4 135.9 140.8 1.9
13.|-- 104.44.4.77 0.0% 5 136.7 136.9 136.0 137.6 0.5
14.|-- 104.44.4.52 0.0% 5 143.8 138.0 135.7 143.8 3.2
15.|-- 104.44.4.65 0.0% 5 137.7 136.8 135.6 137.7 0.0
16.|-- 104.44.5.8 0.0% 5 136.9 136.6 136.1 137.2 0.0
17.|-- 104.44.7.71 0.0% 5 136.1 136.1 135.8 136.4 0.0
18.|-- 104.44.7.39 0.0% 5 135.9 136.3 135.9 136.5 0.0
19.|-- 104.44.11.82 0.0% 5 135.7 135.4 135.2 135.7 0.0
20.|-- ??? 100.0 5 0.0 0.0 0.0 0.0 0.0
#If you use iptables to restrict outbound connections to specific user accounts you can test
sudo -u username nmap -Pn 1.1.1.1 -p53
sudo -u username nc -v -w2 1.1.1.1 53
#use awk to see connections to local port 9090 and show count of remote ips
sudo netstat -nap | awk '/:9090 / && !/::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c
#Watch changes. Also wrap in $'command \'nested string\' |command' for better escaping of only nested single quotes
watch -d -n2 $'sudo netstat -nap | awk \'/:9090 / && !/::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}\' |sort|uniq -c'
#You can do more with gawk (sorting and counting in one pass)
#The commands below assume you are on a system that listens on 80/443 and then fetches responses
#from other systems (using various ports)
#View client IP count by dst port. Limits to tcp, exludes listen state, requires local dst port of 80/443,
#then counts src -> local port and displays any with a count more than 5
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|443)$/ {split($4,dst,":");split($5,src,":");groups[(src[1] "-> :" dst[2])]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {print groups[g],g}}'
378 10.100.0.11-> :443
365 10.100.0.12-> :443
363 10.100.0.13-> :443
358 10.100.0.14-> :443
357 10.100.0.15-> :443
#The above results show 350-380 connections from (in this case) an upstream WAF/LB cluster with 5 nodes
#establishing pooled connections to our local port 443
#View fetches in flight (or recently finished). Limits to tcp, excludes listen state and local port 80/443
#(rough filter for sockets where we are client not server), and again displays counts greater than 5
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}'
Count of fetches in flight
43 10.0.0.101:80
11 10.0.1.101:80
9 10.0.2.101:80
7 10.0.3.101:80
#This shows the count of connections we have to port 80 on 4 different backend web servers
#Same as above but also break down connection state details
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}'
Count of fetches in flight
43 10.0.0.101:80 TIME_WAIT=27
9 10.0.1.101:80 ESTABLISHED=6 TIME_WAIT=3
8 10.0.2.101:80 ESTABLISHED=6 TIME_WAIT=2
7 10.0.3.101:80 ESTABLISHED=7
#Here TIME_WAIT represents closed connections waiting for any final packets
#(hard coded at 60s... this is NOT tcp_fin_timeout, see https://stackoverflow.com/a/35000966/17373 )
#You can even see those countdown timers using -o on netstat or ss
# See countdown timer for all TIME_WAIT sockets in 10.0.0.0-10.255.255.255
ss --numeric -o state time-wait dst 10.0.0.0/8
NetidRecv-Q Send-Q Local Address:Port Peer Address:Port
tcp 0 0 192.168.100.1:57516 10.0.0.101:80 timer:(timewait,55sec,0)
tcp 0 0 192.168.100.1:57356 10.0.0.101:80 timer:(timewait,25sec,0)
tcp 0 0 192.168.100.1:57334 10.0.0.101:80 timer:(timewait,6.536ms,0)
tcp 0 0 192.168.100.1:57282 10.0.1.101:80 timer:(timewait,12sec,0)
tcp 0 0 192.168.100.1:57418 10.0.2.101:80 timer:(timewait,38sec,0)
#TIME_WAIT does NOT mean there is any issue with the client or server. Especially if tcp_tw_reuse is enable,
#these sockets will get reused for new connections if there won't be any conflict in TCP Segment numbers.
#The above server with more TIME_WAIT may just be an API server that has a bunch of small request/responces,
#and those sockets don't get reused as much.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment