Multi Tool Subdomain Enumeration

multi-subd.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/agmmnn/2aa4a8fd863445bd25648d3f1e1832f7/multi-subd.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zXOxSr2LenLV"
      },
      "source": [
        "#### [![](https://raw.githubusercontent.com/agmmnn/awesome-blender/master/imgs/github.svg) github.com/agmmnn/multisub](https://github.com/agmmnn/multisub)\n",
        "\n",
        "#### Data Content\n",
        "- `.json` result of httpx query. It contains information such as timestamp, hash, webserver, host informations and so on.\n",
        "- `.txt` files contains only urls."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F8kTzwfVlqY4"
      },
      "outputs": [],
      "source": [
        "!yes | sudo add-apt-repository universe &> /dev/null\n",
        "!yes | sudo apt update &> /dev/null\n",
        "!yes | sudo apt install p7zip-full p7zip-rar &> /dev/null\n",
        "!sudo apt-get install libpcap0.8 &> /dev/null"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gu2GUzkIct4l"
      },
      "outputs": [],
      "source": [
        "!wget $(curl -s https://api.github.com/repos/owasp-amass/amass/releases/latest | grep browser_download_url | grep 'Linux_amd64.zip' | head -n 1 | cut -d '\"' -f 4)\n",
        "!yes | unzip amass*.zip -d \"./tools\" && rm amass*.zip\n",
        "\n",
        "!wget $(curl -s https://api.github.com/repos/projectdiscovery/subfinder/releases/latest | grep browser_download_url | grep 'linux_amd64.zip' | head -n 1 | cut -d '\"' -f 4)\n",
        "!yes | unzip subfinder*.zip -d \"./tools\" && rm subfinder*.zip\n",
        "\n",
        "!wget $(curl -s https://api.github.com/repos/projectdiscovery/httpx/releases/latest | grep browser_download_url | grep 'linux_amd64.zip' | head -n 1 | cut -d '\"' -f 4)\n",
        "!yes | unzip httpx*.zip -d \"./tools\" && rm httpx*.zip"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!chmod u=rwx,g=r,o=r ./tools/xray_linux_amd64"
      ],
      "metadata": {
        "id": "rmixs-8elNnu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "11c7d3a4-8738-40e7-89a4-65c06b739f5b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "chmod: cannot access './tools/xray_linux_amd64': No such file or directory\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4djWUYFdO4Yd"
      },
      "outputs": [],
      "source": [
        "domain_list = [\"health-tourism.com\",\"mymeditravel.com\",\"flymedi.com\",\"whatclinic.com\",\"bookimed.com\",\"bookinghealth.com\",\"booking.dentist\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J9TvbK-PQkJ-"
      },
      "source": [
        "#### `1. Domain Enumeration`\n",
        "\n",
        "_Run it after uploading the config files or just delete the config commands(-config config.ini, -pc provider-config.yaml)._"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gN_FSlNHrUiV"
      },
      "outputs": [],
      "source": [
        "import subprocess\n",
        "import time\n",
        "from concurrent.futures import ProcessPoolExecutor\n",
        "\n",
        "start_total_time = time.perf_counter()\n",
        "\n",
        "# Create a ProcessPoolExecutor with 4 workers\n",
        "with ProcessPoolExecutor(max_workers=4) as executor:\n",
        "    future_list = []\n",
        "    for domain in domain_list:\n",
        "        start_time = time.perf_counter()\n",
        "        print(f\"📄{domain}.amass.txt, {domain}.subfinder.txt, {domain}.xray.txt\")\n",
        "        #!./tools/amass_Linux_amd64/amass enum -passive -d \"$domain\" -o \"$domain\".amass.txt -silent -config \"config.ini\" &> /dev/null\n",
        "        amass_cmd = [\n",
        "            \"./tools/amass_Linux_amd64/amass\",\n",
        "            \"enum\",\n",
        "            \"-passive\",\n",
        "            \"-d\",\n",
        "            domain,\n",
        "            \"-o\",\n",
        "            f\"{domain}.amass.txt\",\n",
        "            \"-silent\",\n",
        "            \"-config\",\n",
        "            \"./config.ini\",\n",
        "        ]\n",
        "        #!./tools/subfinder -d $domain -o \"$domain\".subfinder.txt -all -t 20 -nW -silent -max-time 120 -timeout 6  -pc \"provider-config.yaml\" &> /dev/null\n",
        "        subfinder_cmd = [\n",
        "            \"./tools/subfinder\",\n",
        "            \"-d\",\n",
        "            domain,\n",
        "            \"-o\",\n",
        "            f\"{domain}.subfinder.txt\",\n",
        "            \"-all\",\n",
        "            \"-t\",\n",
        "            \"20\",\n",
        "            # \"-nW\",\n",
        "            \"-silent\",\n",
        "            \"-max-time\",\n",
        "            \"120\",\n",
        "            \"-timeout\",\n",
        "            \"6\",\n",
        "            \"-pc\",\n",
        "            \"./provider-config.yaml\",\n",
        "        ]\n",
        "\n",
        "        xray_cmd = [\n",
        "            \"./tools/xray_linux_amd64\",\n",
        "            \"sd\",\n",
        "            \"x\",\n",
        "            \"-t\",\n",
        "            domain,\n",
        "            \"--text-output\",\n",
        "            f\"{domain}.xray.txt\",\n",
        "            \"--no-brute\"\n",
        "        ]\n",
        "\n",
        "        #Submit the subprocesses to the executor and store the future objects in a list\n",
        "        future_list.append(executor.submit(subprocess.run, \" \".join(amass_cmd),shell=True))\n",
        "        future_list.append(executor.submit(subprocess.run, \" \".join(subfinder_cmd),shell=True))\n",
        "        future_list.append(executor.submit(subprocess.run, \" \".join(xray_cmd),shell=True))\n",
        "\n",
        "    # Wait for all the submitted subprocesses to complete\n",
        "    for future in future_list:\n",
        "        future.result()\n",
        "\n",
        "    end_time = time.perf_counter()\n",
        "    execution_time = end_time - start_time\n",
        "    print(f\"Execution time: {int(execution_time)}s\")\n",
        "\n",
        "end_total_time = time.perf_counter()\n",
        "execution_total_time = end_total_time - start_total_time\n",
        "print(f\"\\nTotal execution time: {int(execution_total_time)}s\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KJX02fyeQv0A"
      },
      "source": [
        "#### `2. Merge lists, clean non-root domains`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ByRzWey_epu0"
      },
      "outputs": [],
      "source": [
        "for domain in domain_list:\n",
        "    # merge files/lists\n",
        "    inputlist = []\n",
        "    inputfiles = [domain + \".amass.txt\", domain + \".subfinder.txt\"]\n",
        "    for i in inputfiles:\n",
        "        with open(i, \"r\", encoding=\"utf-8\") as f:\n",
        "            inputlist.extend(f.readlines())\n",
        "    with open(domain + \".xray.txt\", \"r\", encoding=\"utf-8\") as f:\n",
        "        xlist=f.readlines()\n",
        "        for x in xlist:\n",
        "            inputlist.append(x.split(\",\")[0]+\"\\n\")\n",
        "\n",
        "\n",
        "    inputlist=list(set(inputlist))\n",
        "    with open(domain + \"_all_final.txt\", \"w\", encoding=\"utf-8\") as f:\n",
        "        for i in inputlist:\n",
        "            f.write(i)\n",
        "    print(f\"💾{domain}_all_final.txt: {len(inputlist)}\")\n",
        "\n",
        "    # cleaning to root domains\n",
        "    itemlist = []\n",
        "    for i in inputlist:\n",
        "        if \"xn--\" in i:\n",
        "            continue\n",
        "        item = i.split(f\".{domain}\")[0].split(\".\")[-1] + f\".{domain}\"\n",
        "        if item not in itemlist:\n",
        "            itemlist.append(item)\n",
        "\n",
        "    # save domains: domain_final.txt\n",
        "    itemlist.sort()\n",
        "    with open(domain + \"_final.txt\", \"w\", encoding=\"utf-8\") as f:\n",
        "        if not domain == \"tsk.tr\":\n",
        "            for i in itemlist:\n",
        "                f.write(f\"{i}\\n\")\n",
        "        else:\n",
        "            for i in itemlist:\n",
        "                if i != \"www.tsk.tr\":\n",
        "                    f.write(f\"www.{i}\\n\")\n",
        "                else:\n",
        "                    f.write(f\"{i}\\n\")\n",
        "    print(f\"💾{domain}_final.txt: {len(itemlist)}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_CvB3nTdh2V"
      },
      "source": [
        "#### `3. Validating the domain list with httpx`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vOXL7N0G3nPG"
      },
      "outputs": [],
      "source": [
        "from concurrent.futures import ProcessPoolExecutor\n",
        "\n",
        "child_processes = []\n",
        "\n",
        "for domain in domain_list:\n",
        "    print(f\"📄{domain}_all_final.jsonl\")\n",
        "    p = subprocess.Popen(\n",
        "        [\n",
        "            \"./tools/httpx\",\n",
        "            \"-l\",\n",
        "            f\"{domain}_all_final.txt\",\n",
        "            \"-t\",\n",
        "            \"100\",\n",
        "            \"-timeout\",\n",
        "            \"7\",\n",
        "            \"-json\",\n",
        "            \"-o\",\n",
        "            f\"{domain}_final.jsonl\",\n",
        "            \"-silent\",\n",
        "            \"-td\"\n",
        "        ],\n",
        "        stdout=subprocess.DEVNULL\n",
        "    )\n",
        "    child_processes.append(p)\n",
        "\n",
        "with ProcessPoolExecutor(max_workers=3) as executor:\n",
        "    for cp in child_processes:\n",
        "        executor.submit(cp.wait)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xtnn9onyRPl8"
      },
      "source": [
        "#### `4. Generating json file and url list from httpx output`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MrxsqyWp6DfJ"
      },
      "outputs": [],
      "source": [
        "for domain in domain_list:\n",
        "    import json\n",
        "\n",
        "    with open(domain + \"_final.jsonl\", encoding=\"utf-8\") as f:\n",
        "        data = [json.loads(line) for line in f]\n",
        "\n",
        "    print(f\"🌐{domain}:\", len(data))\n",
        "    out = json.dumps(data)  # .encode(\"utf-8\").decode('unicode-escape')\n",
        "\n",
        "    with open(domain + \".json\", \"w\", encoding=\"utf-8\") as f:\n",
        "        f.write(out)\n",
        "    print(f\"\\t💾 {domain}.json\")\n",
        "\n",
        "    # json to urls.txt\n",
        "    with open(domain + \".json\", \"r\", encoding=\"utf-8\") as f:\n",
        "        jdata = json.load(f)\n",
        "\n",
        "    urls = []\n",
        "    for i in jdata:\n",
        "        item = i[\"url\"].split(\"//\")[1].split(\":\")[0]\n",
        "        if item not in urls:\n",
        "            urls.append(item)\n",
        "    urls.sort()\n",
        "    with open(domain + \".txt\", \"w\", encoding=\"utf-8\") as f:\n",
        "        for i in urls:\n",
        "            f.write(\"%s\\n\" % i)\n",
        "    print(f\"\\t💾 {domain}.txt\")\n",
        "\n",
        "# final zip\n",
        "import datetime\n",
        "final_files = (\n",
        "    \".txt \".join(domain_list) + \".txt\" + \" \" + \".json \".join(domain_list) + \".json\"\n",
        ")\n",
        "all_files=\"_final.txt \".join(domain_list) + \"_final.txt\" + \" \" + \"_all_final.txt \".join(domain_list) + \"_all_final.txt\" + \" \" + final_files\n",
        "date = datetime.datetime.now().strftime(\"%Y_%m_%d\")\n",
        "zipfile = f\"{date}_final.7z\"\n",
        "zipallfile = f\"{date}_final_all.7z\"\n",
        "print(f\"\\n🗃️ {zipfile}\")\n",
        "!7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on $zipfile $final_files &> /dev/null\n",
        "print(f\"🗃️ {zipallfile}\")\n",
        "!7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on $zipallfile $all_files &> /dev/null"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}