mhmdiaa

cat urls.txt | httpx -json -store-response -output httpx.json

cat httpx.json | jq -r '"\(.stored_response_path) \(.path | ltrimstr("/"))"' | xargs -n 2 sh -c 'mkdir -p "$(dirname $2)" && cp $1 $2' sh

mhmdiaa

javascript: (function() {
    var scripts = document.getElementsByTagName("script"),
        regex = /(?<=(\"|\%27|\`))\/[a-zA-Z0-9_?&=\/\-\#\.]*(?=(\"|\'|\%60))/g;
    const results = new Set;
    for (var i = 0; i < scripts.length; i++) {
        var t = scripts[i].src;
        "" != t && fetch(t).then(function(t) {
            return t.text()
        }).then(function(t) {
            var e = t.matchAll(regex);

mhmdiaa

Keybase proof

I hereby claim:

• I am mhmdiaa on github.
• I am mhmdiaa (https://keybase.io/mhmdiaa) on keybase.
• I have a public key ASB7gQF1QTGIH2fnl6BL88-73BM1Kz-DFfx4acc7aWAYawo

To claim this, I am signing this object:

mhmdiaa

import sys


def to_octets(ip):
    return [int(i) for i in ip.split('.')]


def dotless_decimal(ip):
    octets = to_octets(ip)
    result = octets[0] * 16777216 + octets[1] * \

mhmdiaa

import requests
import sys
import json


def waybackurls(host, with_subs):
    if with_subs:
        url = 'http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey' % host
    else:
        url = 'http://web.archive.org/cdx/search/cdx?url=%s/*&output=json&fl=original&collapse=urlkey' % host

mhmdiaa

import requests
import re
import sys
from multiprocessing.dummy import Pool


def robots(host):
    r = requests.get(
        'https://web.archive.org/cdx/search/cdx\
        ?url=%s/robots.txt&output=json&fl=timestamp,original&filter=statuscode:200&collapse=digest' % host)