gbrayut/test.sh

## test.sh
#Test all A entries for a DNS record to see if you can connect to port 443:
dig +short microsoft.com. | xargs -I {} nc -v -w2 {} 443

#Example of output
Connection to 191.239.213.197 443 port [tcp/https] succeeded!
Connection to 104.43.195.251 443 port [tcp/https] succeeded!
Connection to 104.40.211.35 443 port [tcp/https] succeeded!
Connection to 23.100.122.175 port 443 (tcp) timed out: Operation now in progress
Connection to 23.96.52.53 443 port [tcp/https] succeeded!


#Test using a string as input (printf prevents issue with \n from echo. echo -n also works)
printf %b '10.1.0.1 10.2.0.2 10.3.0.3 10.4.0.4' | xargs -d ' ' -I {} nc -v -w2 {} 443
echo -n test-{east,west,south,north}-server.example.com | xargs -d ' ' -I {} nc -v -w2 {} 443

#Example of output
nc: connect to 10.1.0.1 443 port [tcp/https] succeeded!
nc: connect to 10.2.0.2 port 443 (tcp) failed: Connection refused
nc: connect to 10.3.0.3 port 443 (tcp) timed out: Operation now in progress
nc: connect to 10.4.0.4 port 443 (tcp) timed out: Operation now in progress

#Run multiple commands. This brace expands to 10.x.0.1, .5, and .9 for x=1 thru 10
echo -n 10.{1..10}.0.{1,5,9} | xargs -d ' ' -I {} bash -c "echo -e '\n\n\nIP: {}'; curl -vsm 3 http://{}/"

#Can also test for local route details (example: see which local interface would be used for egress)
printf %b '192.168.0.4 192.168.76.36 192.168.4.100 10.120.100.1' | xargs -d ' ' -I {} ip route get {}
local 192.168.0.4 dev lo  src 192.168.0.4
    cache <local>
192.168.76.36 via 192.168.0.1 dev enp0s31f6  src 192.168.0.4
    cache
192.168.4.100 dev lxdbr0  src 192.168.4.1
    cache <local>
10.120.100.1 dev vpn0  src 172.16.2.1
    cache


#Use mtr for traceroute, send output to stdout so you can copy/paste. See also --tcp or --port options if ICMP is blocked
sudo mtr -n -r -c 5 microsoft.com
Start: Thu May 31 12:12:12 2018
HOST: gbmint02                    Loss%   Snt   Last   Avg  Best  Wrst StDev
  1.|-- 192.168.0.1                0.0%     5    0.4   0.4   0.4   0.4   0.0
  2.|-- 96.120.96.89               0.0%     5    8.1   7.7   7.3   8.1   0.0
  3.|-- 68.87.170.221              0.0%     5    7.6   7.7   7.6   7.9   0.0
  4.|-- 69.139.231.85              0.0%     5    7.8   7.9   7.6   8.3   0.0
  5.|-- 68.86.90.225               0.0%     5   20.0  20.4  19.9  21.1   0.0
  6.|-- 68.86.84.226               0.0%     5   20.6  29.6  20.2  64.6  19.6
  7.|-- 68.86.83.94                0.0%     5   20.0  19.7  19.0  21.3   0.9
  8.|-- 50.242.149.162             0.0%     5   19.0  19.7  19.0  21.1   0.5
  9.|-- 104.44.4.247               0.0%     5  138.1 138.0 136.0 140.1   1.4
 10.|-- 104.44.4.110               0.0%     5  138.6 137.9 135.6 140.1   1.6
 11.|-- 104.44.4.96                0.0%     5  136.2 136.9 136.2 137.8   0.5
 12.|-- 104.44.4.175               0.0%     5  135.9 137.4 135.9 140.8   1.9
 13.|-- 104.44.4.77                0.0%     5  136.7 136.9 136.0 137.6   0.5
 14.|-- 104.44.4.52                0.0%     5  143.8 138.0 135.7 143.8   3.2
 15.|-- 104.44.4.65                0.0%     5  137.7 136.8 135.6 137.7   0.0
 16.|-- 104.44.5.8                 0.0%     5  136.9 136.6 136.1 137.2   0.0
 17.|-- 104.44.7.71                0.0%     5  136.1 136.1 135.8 136.4   0.0
 18.|-- 104.44.7.39                0.0%     5  135.9 136.3 135.9 136.5   0.0
 19.|-- 104.44.11.82               0.0%     5  135.7 135.4 135.2 135.7   0.0
 20.|-- ???                       100.0     5    0.0   0.0   0.0   0.0   0.0

#If you use iptables to restrict outbound connections to specific user accounts you can test
sudo -u username nmap -Pn 1.1.1.1 -p53
sudo -u username nc -v -w2 1.1.1.1 53

## zz-lb-proxy-sockets.sh
#use awk to see connections to local port 9090 and show count of remote ips
sudo netstat -nap | awk '/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c
#similar, but only include lines with local ip:80 or :443
sudo netstat -nap | awk "/$(hostname -i):(80|443) / && ! /::1:| LISTEN / {match(\$5, /(.*):(.*)/, m); print m[1]}" |sort|uniq -c
#Watch changes. Also wrap in $'command \'nested string\' |command' for better escaping of only nested single quotes
watch -d -n2 $'sudo netstat -nap | awk \'/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}\' |sort|uniq -c'
#Watch local 80/443, excluding TIME_WAIT (rough estimate of active connections to local webserver). Change m[1] to \$0 to see raw data.
watch -d -n2 "sudo netstat -nap | awk '/$(hostname -i):(80|443) / && ! /::1:| LISTEN | TIME_WAIT/ {match(\$5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c"

#You can do more with gawk (sorting and counting in one pass)
#The commands below assume you are on a system that listens on 80/443 and then fetches responses
#from other systems (using various ports)

#View client IP count by dst port. Limits to tcp, exludes listen state, requires local dst port of 80/443,
#then counts src -> local port and displays any with a count more than 5 along with socket counts
uptime;netstat -na > /dev/shm/netstat; echo Total Established $(grep -P '(?<!127\.0\.0\.1):4444 .*ESTABLISHED' /dev/shm/netstat | wc -l); gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|443)$/ {split($4,dst,":");split($5,src,":");groups[(src[1] "-> :" dst[2])]++;states[(src[1] "-> :" dst[2])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat

13:05:09 up 1 day,  5:13,  1 user,  load average: 1.06, 0.66, 0.66
Total Established 106
378     10.100.0.11-> :443    ESTABLISHED=300 TIME_WAIT=78
365     10.100.0.12-> :443    TIME_WAIT=250 ESTABLISHED=115
363     10.100.0.13-> :443    ESTABLISHED=363
358     10.100.0.14-> :443    TIME_WAIT=358
357     10.100.0.15-> :443    ESTABLISHED=200 TIME_WAIT=157

#The above results show 350-380 connections from (in this case) an upstream WAF/LB cluster with 5 nodes
#establishing pooled connections to our local port 443

# Another for loop to monitor a go server in a container (fixes issue with ipv6 source addresses):
for run in `seq 1 10`; do uptime;netstat -nat > /dev/shm/netstat; echo "Totals: $(curl -s 127.0.0.1:80/metrics|grep '^go_goroutines')       Established $(grep 'ESTABLISHED' /dev/shm/netstat | wc -l)"; gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|3000|3001)$/{ndst=split($4,dst,":");nsrc=split($5,src,":");groups[(src[nsrc-1] "-> :" dst[ndst])]++;states[(src[nsrc-1] "-> :" dst[ndst])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat; sleep 600;done;

#View fetches in flight (or recently finished). Limits to tcp, excludes listen state and local port 80/443
#(rough filter for sockets where we are client not server), and again displays counts greater than 5
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}'

Count of fetches in flight
43      10.0.0.101:80
11      10.0.1.101:80
9       10.0.2.101:80
7       10.0.3.101:80
#Same as above but in format for watch or remote salt cmd.run
watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}\''

#This shows the count of connections we have to port 80 on 4 different backend web servers

#Same as above but also break down connection state details
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}'
Count of fetches in flight
43      10.0.0.101:80  TIME_WAIT=27
9       10.0.1.101:80   ESTABLISHED=6 TIME_WAIT=3
8       10.0.2.101:80  ESTABLISHED=6 TIME_WAIT=2
7       10.0.3.101:80   ESTABLISHED=7
#Same as above but in format for watch or remote salt cmd.run
watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}\''

#Here TIME_WAIT represents closed connections waiting for any final packets
#(hard coded at 60s... this is NOT tcp_fin_timeout, see https://stackoverflow.com/a/35000966/17373 )
#You can even see those countdown timers using -o on netstat or ss

# See countdown timer for all TIME_WAIT sockets in 10.0.0.0-10.255.255.255
ss --numeric -o state time-wait dst 10.0.0.0/8

NetidRecv-Q  Send-Q    Local Address:Port    Peer Address:Port
tcp  0       0         192.168.100.1:57516   10.0.0.101:80    timer:(timewait,55sec,0)
tcp  0       0         192.168.100.1:57356   10.0.0.101:80    timer:(timewait,25sec,0)
tcp  0       0         192.168.100.1:57334   10.0.0.101:80    timer:(timewait,6.536ms,0)
tcp  0       0         192.168.100.1:57282   10.0.1.101:80    timer:(timewait,12sec,0)
tcp  0       0         192.168.100.1:57418   10.0.2.101:80    timer:(timewait,38sec,0)

#TIME_WAIT does NOT mean there is any issue with the client or server. Especially if tcp_tw_reuse is enable,
#these sockets will get reused for new connections if there won't be any conflict in TCP Segment numbers.

#The above server with more TIME_WAIT may just be an API server that has a bunch of small request/responces,
#and those sockets don't get reused as much.
	#Test all A entries for a DNS record to see if you can connect to port 443:
	dig +short microsoft.com. \| xargs -I {} nc -v -w2 {} 443

	#Example of output
	Connection to 191.239.213.197 443 port [tcp/https] succeeded!
	Connection to 104.43.195.251 443 port [tcp/https] succeeded!
	Connection to 104.40.211.35 443 port [tcp/https] succeeded!
	Connection to 23.100.122.175 port 443 (tcp) timed out: Operation now in progress
	Connection to 23.96.52.53 443 port [tcp/https] succeeded!


	#Test using a string as input (printf prevents issue with \n from echo. echo -n also works)
	printf %b '10.1.0.1 10.2.0.2 10.3.0.3 10.4.0.4' \| xargs -d ' ' -I {} nc -v -w2 {} 443
	echo -n test-{east,west,south,north}-server.example.com \| xargs -d ' ' -I {} nc -v -w2 {} 443

	#Example of output
	nc: connect to 10.1.0.1 443 port [tcp/https] succeeded!
	nc: connect to 10.2.0.2 port 443 (tcp) failed: Connection refused
	nc: connect to 10.3.0.3 port 443 (tcp) timed out: Operation now in progress
	nc: connect to 10.4.0.4 port 443 (tcp) timed out: Operation now in progress

	#Run multiple commands. This brace expands to 10.x.0.1, .5, and .9 for x=1 thru 10
	echo -n 10.{1..10}.0.{1,5,9} \| xargs -d ' ' -I {} bash -c "echo -e '\n\n\nIP: {}'; curl -vsm 3 http://{}/"

	#Can also test for local route details (example: see which local interface would be used for egress)
	printf %b '192.168.0.4 192.168.76.36 192.168.4.100 10.120.100.1' \| xargs -d ' ' -I {} ip route get {}
	local 192.168.0.4 dev lo src 192.168.0.4
	cache <local>
	192.168.76.36 via 192.168.0.1 dev enp0s31f6 src 192.168.0.4
	cache
	192.168.4.100 dev lxdbr0 src 192.168.4.1
	cache <local>
	10.120.100.1 dev vpn0 src 172.16.2.1
	cache


	#Use mtr for traceroute, send output to stdout so you can copy/paste. See also --tcp or --port options if ICMP is blocked
	sudo mtr -n -r -c 5 microsoft.com
	Start: Thu May 31 12:12:12 2018
	HOST: gbmint02 Loss% Snt Last Avg Best Wrst StDev
	1.\|-- 192.168.0.1 0.0% 5 0.4 0.4 0.4 0.4 0.0
	2.\|-- 96.120.96.89 0.0% 5 8.1 7.7 7.3 8.1 0.0
	3.\|-- 68.87.170.221 0.0% 5 7.6 7.7 7.6 7.9 0.0
	4.\|-- 69.139.231.85 0.0% 5 7.8 7.9 7.6 8.3 0.0
	5.\|-- 68.86.90.225 0.0% 5 20.0 20.4 19.9 21.1 0.0
	6.\|-- 68.86.84.226 0.0% 5 20.6 29.6 20.2 64.6 19.6
	7.\|-- 68.86.83.94 0.0% 5 20.0 19.7 19.0 21.3 0.9
	8.\|-- 50.242.149.162 0.0% 5 19.0 19.7 19.0 21.1 0.5
	9.\|-- 104.44.4.247 0.0% 5 138.1 138.0 136.0 140.1 1.4
	10.\|-- 104.44.4.110 0.0% 5 138.6 137.9 135.6 140.1 1.6
	11.\|-- 104.44.4.96 0.0% 5 136.2 136.9 136.2 137.8 0.5
	12.\|-- 104.44.4.175 0.0% 5 135.9 137.4 135.9 140.8 1.9
	13.\|-- 104.44.4.77 0.0% 5 136.7 136.9 136.0 137.6 0.5
	14.\|-- 104.44.4.52 0.0% 5 143.8 138.0 135.7 143.8 3.2
	15.\|-- 104.44.4.65 0.0% 5 137.7 136.8 135.6 137.7 0.0
	16.\|-- 104.44.5.8 0.0% 5 136.9 136.6 136.1 137.2 0.0
	17.\|-- 104.44.7.71 0.0% 5 136.1 136.1 135.8 136.4 0.0
	18.\|-- 104.44.7.39 0.0% 5 135.9 136.3 135.9 136.5 0.0
	19.\|-- 104.44.11.82 0.0% 5 135.7 135.4 135.2 135.7 0.0
	20.\|-- ??? 100.0 5 0.0 0.0 0.0 0.0 0.0

	#If you use iptables to restrict outbound connections to specific user accounts you can test
	sudo -u username nmap -Pn 1.1.1.1 -p53
	sudo -u username nc -v -w2 1.1.1.1 53
	#use awk to see connections to local port 9090 and show count of remote ips
	sudo netstat -nap \| awk '/:9090 / && ! /::1:\| LISTEN / {match($5, /(.):(.)/, m); print m[1]}' \|sort\|uniq -c
	#similar, but only include lines with local ip:80 or :443
	sudo netstat -nap \| awk "/$(hostname -i):(80\|443) / && ! /::1:\| LISTEN / {match(\$5, /(.):(.)/, m); print m[1]}" \|sort\|uniq -c
	#Watch changes. Also wrap in $'command \'nested string\' \|command' for better escaping of only nested single quotes
	watch -d -n2 $'sudo netstat -nap \| awk \'/:9090 / && ! /::1:\| LISTEN / {match($5, /(.):(.)/, m); print m[1]}\' \|sort\|uniq -c'
	#Watch local 80/443, excluding TIME_WAIT (rough estimate of active connections to local webserver). Change m[1] to \$0 to see raw data.
	watch -d -n2 "sudo netstat -nap \| awk '/$(hostname -i):(80\|443) / && ! /::1:\| LISTEN \| TIME_WAIT/ {match(\$5, /(.):(.)/, m); print m[1]}' \|sort\|uniq -c"

	#You can do more with gawk (sorting and counting in one pass)
	#The commands below assume you are on a system that listens on 80/443 and then fetches responses
	#from other systems (using various ports)

	#View client IP count by dst port. Limits to tcp, exludes listen state, requires local dst port of 80/443,
	#then counts src -> local port and displays any with a count more than 5 along with socket counts
	uptime;netstat -na > /dev/shm/netstat; echo Total Established $(grep -P '(?<!127\.0\.0\.1):4444 .*ESTABLISHED' /dev/shm/netstat \| wc -l); gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80\|443)$/ {split($4,dst,":");split($5,src,":");groups[(src[1] "-> :" dst[2])]++;states[(src[1] "-> :" dst[2])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat

	13:05:09 up 1 day, 5:13, 1 user, load average: 1.06, 0.66, 0.66
	Total Established 106
	378 10.100.0.11-> :443 ESTABLISHED=300 TIME_WAIT=78
	365 10.100.0.12-> :443 TIME_WAIT=250 ESTABLISHED=115
	363 10.100.0.13-> :443 ESTABLISHED=363
	358 10.100.0.14-> :443 TIME_WAIT=358
	357 10.100.0.15-> :443 ESTABLISHED=200 TIME_WAIT=157

	#The above results show 350-380 connections from (in this case) an upstream WAF/LB cluster with 5 nodes
	#establishing pooled connections to our local port 443

	# Another for loop to monitor a go server in a container (fixes issue with ipv6 source addresses):
	for run in `seq 1 10`; do uptime;netstat -nat > /dev/shm/netstat; echo "Totals: $(curl -s 127.0.0.1:80/metrics\|grep '^go_goroutines') Established $(grep 'ESTABLISHED' /dev/shm/netstat \| wc -l)"; gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80\|3000\|3001)$/{ndst=split($4,dst,":");nsrc=split($5,src,":");groups[(src[nsrc-1] "-> :" dst[ndst])]++;states[(src[nsrc-1] "-> :" dst[ndst])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat; sleep 600;done;

	#View fetches in flight (or recently finished). Limits to tcp, excludes listen state and local port 80/443
	#(rough filter for sockets where we are client not server), and again displays counts greater than 5
	netstat -na \| gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80\|443)$/ {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}'

	Count of fetches in flight
	43 10.0.0.101:80
	11 10.0.1.101:80
	9 10.0.2.101:80
	7 10.0.3.101:80
	#Same as above but in format for watch or remote salt cmd.run
	watch -d -n5 $'netstat -na \| gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80\|443)$/) {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}\''

	#This shows the count of connections we have to port 80 on 4 different backend web servers

	#Same as above but also break down connection state details
	netstat -na \| gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80\|443)$/ {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}'
	Count of fetches in flight
	43 10.0.0.101:80 TIME_WAIT=27
	9 10.0.1.101:80 ESTABLISHED=6 TIME_WAIT=3
	8 10.0.2.101:80 ESTABLISHED=6 TIME_WAIT=2
	7 10.0.3.101:80 ESTABLISHED=7
	#Same as above but in format for watch or remote salt cmd.run
	watch -d -n5 $'netstat -na \| gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80\|443)$/) {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}\''

	#Here TIME_WAIT represents closed connections waiting for any final packets
	#(hard coded at 60s... this is NOT tcp_fin_timeout, see https://stackoverflow.com/a/35000966/17373 )
	#You can even see those countdown timers using -o on netstat or ss

	# See countdown timer for all TIME_WAIT sockets in 10.0.0.0-10.255.255.255
	ss --numeric -o state time-wait dst 10.0.0.0/8

	NetidRecv-Q Send-Q Local Address:Port Peer Address:Port
	tcp 0 0 192.168.100.1:57516 10.0.0.101:80 timer:(timewait,55sec,0)
	tcp 0 0 192.168.100.1:57356 10.0.0.101:80 timer:(timewait,25sec,0)
	tcp 0 0 192.168.100.1:57334 10.0.0.101:80 timer:(timewait,6.536ms,0)
	tcp 0 0 192.168.100.1:57282 10.0.1.101:80 timer:(timewait,12sec,0)
	tcp 0 0 192.168.100.1:57418 10.0.2.101:80 timer:(timewait,38sec,0)

	#TIME_WAIT does NOT mean there is any issue with the client or server. Especially if tcp_tw_reuse is enable,
	#these sockets will get reused for new connections if there won't be any conflict in TCP Segment numbers.

	#The above server with more TIME_WAIT may just be an API server that has a bunch of small request/responces,
	#and those sockets don't get reused as much.