crawl a web page, extract all domains and resolve them to IP addresses with bash and common GNU/Linux tools
#!/bin/bash | |
url='rawsec.ml' | |
domains=$(curl $url -s | grep -E 'https?://[^"]*' | cut -d '/' -f 3 | cut -d '"' -f 1 | uniq) | |
filename='/tmp/temporary_ips.txt' | |
for domain in $domains | |
do | |
dig +noall +answer $domain | awk '/\sA\s/ {print $5}' >> $filename | |
done | |
cat $filename | sort -u |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
ASCII video demo: https://asciinema.org/a/243061
article: https://rawsec.ml/en/crawl-web-page-domains-resolve-ips/