$ cat subdo.txt | hakrawler | grep 'http' | cut -d '' -f 2 > crawler.txt
$ gau -subs domain.com >> crawler.txt
$ waybackurls domain.com >> crawler.txt
$ cat crawling.txt | grep "?" | unfurl --unique format %s://%d%p > base.txt
$ cat base.txt | parallel -j50 -q grep {} -m5 crawling.txt | tee -a final.txt
$ cat final.txt | egrep -iv ".(jpg|jpeg|gif|css|tif|tiff|woff|woff2|ico|pdf|svg|txt|js)" > final_bos.txt
$ rm -rf base.txt final.txt
# CommonCrawl to find endpoints on a site
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| curl -s "http://web.archive.org/cdx/search/cdx?url=*.domain/*&output=txt&fl=original&collapse=urlkey&page=/" | gf ssrf | qsreplace "https://localtest.me" | parallel -j50 -q curl -i -s -k -o >(grep -io "<title>[^<]*" | cut -d'>' -f2-) --silent --max-time 2 --write-out 'Status:%{http_code}\t Header-size:%{size_header}tUrl:%{url_effective} || ' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Execute as wget -O - https://gist.githubusercontent.com/ravkishu/a45139044afb8b3a39542f0c4e42ba8a/raw | bash | |
| # It's debian based, so for centos and likewise you have to change apt to yum and similarly | |
| # | |
| InstallationStartTime=$(date +%s) | |
| #### COLORS #### ( Taken from : https://misc.flogisoft.com/bash/tip_colors_and_formatting ) | |
| NORMAL='\e[0m' | |
| RED='\e[31m' | |
| LIGHT_GREEN='\e[92m' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| import requests | |
| import time | |
| import os | |
| import json | |
| import sys | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0', | |
| 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "proxy":{ | |
| "ssl_pass_through":{ | |
| "automatically_add_entries_on_client_ssl_negotiation_failure":false, | |
| "rules":[ | |
| { | |
| "enabled":true, | |
| "host":".*\\.google\\.com", | |
| "protocol":"any" | |
| }, |
NewerOlder