generate wordlists utilizing the wayback machine

waywords.go

package main

import (
	"bufio"
	"fmt"
	"net/url"
	"os"
	"strings"
)

func main() {
	// usage: printf "example.com" | waybackurls | waywords
	words := make(map[string]struct{})
	w := bufio.NewScanner(os.Stdin)
	for w.Scan() {
		u, err := url.Parse(w.Text())
		if err != nil {
			continue
		}
		for _, t := range strings.Split(u.Path, "/") {
			if t == "" {
				continue
			}
			words[t] = struct{}{}
		}
	}
	for word, _ := range words {
		fmt.Println(word)
	}
}

echo example.com | waybackurls | unfurl -u paths | sed -r 's#/#\n#g' | sort -u 🙃

echo example.com | waybackurls | unfurl -u paths | sed -r 's#/#\n#g' | sort -u 🙃

Is that so, Mr. Hudson.

curl -s 'http://web.archive.org/cdx/search/cdx?collapse=urlkey&url=*.hacker.one' | awk -F'[) ?&]' 'gsub("/", "\n"){print $2 | "sort -u"}'

😆

waybackurls example.com | cut -d/ -f4- | sort -u

🐸

vim 'http://web.archive.org/cdx/search/cdx?collapse=urlkey&url=*.hacker.one' +'%s/.\+)\/\([^ &?]*\).\+/\1/' +'%s/\//\r/' +'sort u'

waybackurls example.com | cut -d/ -f4- | sort -u

🐸

Relying on external tools, Ed? Why don't you install Xorg as well!