jthack

# Before the first run, read the comments and change the script for your company
# Before anything, install ffuf with `go install github.com/ffuf/ffuf@latest`

# Change the WEBROOT variable below to the location of the webroot
WEBROOT=/var/www/html/CHANGE/ME
# This changes to the webroot directory
cd $WEBROOT
# This makes a directory for storing the files used for this script. Change it to be whatever path you want. 
PROJPATH=/home/changeme/project
mkdir -p $PROJPATH

jthack

import requests
import re
import sys
from multiprocessing.dummy import Pool


def robots(host):
    r = requests.get(
        'https://web.archive.org/cdx/search/cdx\
        ?url=%s/robots.txt&output=json&fl=timestamp,original&filter=statuscode:200&collapse=digest' % host)

jthack

import requests
import sys
import json


def waybackurls(host, with_subs):
    if with_subs:
        url = 'http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey' % host
    else:
        url = 'http://web.archive.org/cdx/search/cdx?url=%s/*&output=json&fl=original&collapse=urlkey' % host