debian@pikpikcu~$ cat subdo.txt | hakrawler | grep 'http' | cut -d '' -f 2 > crawler.txt
debian@pikpikcu~$ gau -subs domain.com >> crawler.txt
debian@pikpikcu~$ waybackurls domain.com >> crawler.txt
debian@pikpikcu~$ cat crawling.txt | grep "?" | unfurl --unique format %s://%d%p > base.txt
debian@pikpikcu~$ cat base.txt | parallel -j50 -q grep {} -m5 crawling.txt | tee -a final.txt
debian@pikpikcu~$ cat final.txt | egrep -iv ".(jpg|jpeg|gif|css|tif|tiff|woff|woff2|ico|pdf|svg|txt|js)" > final_bos.txt
debian@pikpikcu~$ rm -rf base.txt final.txt
Last active
July 8, 2022 21:12
-
-
Save pikpikcu/99857901b7937840127966936647cff0 to your computer and use it in GitHub Desktop.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment