Skip to content

Instantly share code, notes, and snippets.

@Sh1n0g1

Sh1n0g1/shodan-query.ipynb

Last active July 21, 2023 04:52
Show Gist options
  • Save Sh1n0g1/1164800efdbdd29ee87220cd78f2157c to your computer and use it in GitHub Desktop.
Save Sh1n0g1/1164800efdbdd29ee87220cd78f2157c to your computer and use it in GitHub Desktop.
Download ZIP
Shodan Query.ipynb
Display the source blob
Display the rendered blob
Raw
shodan-query.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"authorship_tag": "ABX9TyOR42Jj4cCNabLofkEeGcs5",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/Sh1n0g1/1164800efdbdd29ee87220cd78f2157c/shodan-query.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"source": [
"# Get Massive Result from SHODAN\n",
"\n",
"## Problem\n",
"* We can download 100 hosts per requests and use `page` parameter to request the next page.\n",
"* Unfortunately, Shodan API is not really stable when we try to download more than ten thousand records, even the Shodan CLI.\n",
"* They have pagination but when the page number become larger, we get `500` error or suddenly, `wrong query` error.\n",
"* Especially when we try to download all hosts running a popular server/appliance.\n",
"\n",
"## Solution\n",
"* Split the request by netblock and request the query for each A class netblock.\n",
"* Fill the `1. User Input` form and run all cell.\n",
"* Download the result.zip to download the zipped json files.\n",
"\n",
"### If you have multiple queries\n",
"* Fill the `1. User Input` form with the first query\n",
"* Run `2 - 4`\n",
"* Put the next query on `1. User Input` form\n",
"* Run `4` individually.\n",
"* Repeat after you put all queries\n",
"* Lastly, run 5 & 6."
],
"metadata": {
"id": "AjTFPJ0Uuais"
}
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "ci9dCG5luWCt"
},
"outputs": [],
"source": [
"#@title 1.User Input\n",
"SHODAN_API_KEY = \"\" #@param {type:\"string\"}\n",
"shodan_query = \"port:54321\" #@param {type:\"string\"}\n",
"output_directory_name = \"output_directory/\" #@param {type:\"string\"}\n",
"MAX_ERROR_COUNT=100"
]
},
{
"cell_type": "code",
"source": [
"#@title 2. Library & Constant Variable\n",
"import os\n",
"import sys\n",
"import json\n",
"import math\n",
"import time\n",
"import shutil\n",
"import requests\n",
"\n",
"SHODAN_COUNT_URL='https://api.shodan.io/shodan/host/count?key='\n",
"SHODAN_SEARCH_URL='https://api.shodan.io/shodan/host/search?key='\n",
"HOST_PER_MIN=1300\n",
"GLOBAL_NETBLOCKS=[\n",
" \"1.0.0.0/8\",\n",
" \"2.0.0.0/8\",\n",
" \"3.0.0.0/8\",\n",
" \"4.0.0.0/8\",\n",
" \"5.0.0.0/8\",\n",
" \"6.0.0.0/8\",\n",
" \"7.0.0.0/8\",\n",
" \"8.0.0.0/8\",\n",
" \"9.0.0.0/8\",\n",
" \"11.0.0.0/8\",\n",
" \"12.0.0.0/8\",\n",
" \"13.0.0.0/8\",\n",
" \"14.0.0.0/8\",\n",
" \"15.0.0.0/8\",\n",
" \"16.0.0.0/8\",\n",
" \"17.0.0.0/8\",\n",
" \"18.0.0.0/8\",\n",
" \"19.0.0.0/8\",\n",
" \"20.0.0.0/8\",\n",
" \"21.0.0.0/8\",\n",
" \"22.0.0.0/8\",\n",
" \"23.0.0.0/8\",\n",
" \"24.0.0.0/8\",\n",
" \"25.0.0.0/8\",\n",
" \"26.0.0.0/8\",\n",
" \"27.0.0.0/8\",\n",
" \"28.0.0.0/8\",\n",
" \"29.0.0.0/8\",\n",
" \"30.0.0.0/8\",\n",
" \"31.0.0.0/8\",\n",
" \"32.0.0.0/8\",\n",
" \"33.0.0.0/8\",\n",
" \"34.0.0.0/8\",\n",
" \"35.0.0.0/8\",\n",
" \"36.0.0.0/8\",\n",
" \"37.0.0.0/8\",\n",
" \"38.0.0.0/8\",\n",
" \"39.0.0.0/8\",\n",
" \"40.0.0.0/8\",\n",
" \"41.0.0.0/8\",\n",
" \"42.0.0.0/8\",\n",
" \"43.0.0.0/8\",\n",
" \"44.0.0.0/8\",\n",
" \"45.0.0.0/8\",\n",
" \"46.0.0.0/8\",\n",
" \"47.0.0.0/8\",\n",
" \"48.0.0.0/8\",\n",
" \"49.0.0.0/8\",\n",
" \"50.0.0.0/8\",\n",
" \"51.0.0.0/8\",\n",
" \"52.0.0.0/8\",\n",
" \"53.0.0.0/8\",\n",
" \"54.0.0.0/8\",\n",
" \"55.0.0.0/8\",\n",
" \"56.0.0.0/8\",\n",
" \"57.0.0.0/8\",\n",
" \"58.0.0.0/8\",\n",
" \"59.0.0.0/8\",\n",
" \"60.0.0.0/8\",\n",
" \"61.0.0.0/8\",\n",
" \"62.0.0.0/8\",\n",
" \"63.0.0.0/8\",\n",
" \"64.0.0.0/8\",\n",
" \"65.0.0.0/8\",\n",
" \"66.0.0.0/8\",\n",
" \"67.0.0.0/8\",\n",
" \"68.0.0.0/8\",\n",
" \"69.0.0.0/8\",\n",
" \"70.0.0.0/8\",\n",
" \"71.0.0.0/8\",\n",
" \"72.0.0.0/8\",\n",
" \"73.0.0.0/8\",\n",
" \"74.0.0.0/8\",\n",
" \"75.0.0.0/8\",\n",
" \"76.0.0.0/8\",\n",
" \"77.0.0.0/8\",\n",
" \"78.0.0.0/8\",\n",
" \"79.0.0.0/8\",\n",
" \"80.0.0.0/8\",\n",
" \"81.0.0.0/8\",\n",
" \"82.0.0.0/8\",\n",
" \"83.0.0.0/8\",\n",
" \"84.0.0.0/8\",\n",
" \"85.0.0.0/8\",\n",
" \"86.0.0.0/8\",\n",
" \"87.0.0.0/8\",\n",
" \"88.0.0.0/8\",\n",
" \"89.0.0.0/8\",\n",
" \"90.0.0.0/8\",\n",
" \"91.0.0.0/8\",\n",
" \"92.0.0.0/8\",\n",
" \"93.0.0.0/8\",\n",
" \"94.0.0.0/8\",\n",
" \"95.0.0.0/8\",\n",
" \"96.0.0.0/8\",\n",
" \"97.0.0.0/8\",\n",
" \"98.0.0.0/8\",\n",
" \"99.0.0.0/8\",\n",
" \"100.0.0.0/8\",\n",
" \"101.0.0.0/8\",\n",
" \"102.0.0.0/8\",\n",
" \"103.0.0.0/8\",\n",
" \"104.0.0.0/8\",\n",
" \"105.0.0.0/8\",\n",
" \"106.0.0.0/8\",\n",
" \"107.0.0.0/8\",\n",
" \"108.0.0.0/8\",\n",
" \"109.0.0.0/8\",\n",
" \"110.0.0.0/8\",\n",
" \"111.0.0.0/8\",\n",
" \"112.0.0.0/8\",\n",
" \"113.0.0.0/8\",\n",
" \"114.0.0.0/8\",\n",
" \"115.0.0.0/8\",\n",
" \"116.0.0.0/8\",\n",
" \"117.0.0.0/8\",\n",
" \"118.0.0.0/8\",\n",
" \"119.0.0.0/8\",\n",
" \"120.0.0.0/8\",\n",
" \"121.0.0.0/8\",\n",
" \"122.0.0.0/8\",\n",
" \"123.0.0.0/8\",\n",
" \"124.0.0.0/8\",\n",
" \"125.0.0.0/8\",\n",
" \"126.0.0.0/8\",\n",
" \"128.0.0.0/8\",\n",
" \"129.0.0.0/8\",\n",
" \"130.0.0.0/8\",\n",
" \"131.0.0.0/8\",\n",
" \"132.0.0.0/8\",\n",
" \"133.0.0.0/8\",\n",
" \"134.0.0.0/8\",\n",
" \"135.0.0.0/8\",\n",
" \"136.0.0.0/8\",\n",
" \"137.0.0.0/8\",\n",
" \"138.0.0.0/8\",\n",
" \"139.0.0.0/8\",\n",
" \"140.0.0.0/8\",\n",
" \"141.0.0.0/8\",\n",
" \"142.0.0.0/8\",\n",
" \"143.0.0.0/8\",\n",
" \"144.0.0.0/8\",\n",
" \"145.0.0.0/8\",\n",
" \"146.0.0.0/8\",\n",
" \"147.0.0.0/8\",\n",
" \"148.0.0.0/8\",\n",
" \"149.0.0.0/8\",\n",
" \"150.0.0.0/8\",\n",
" \"151.0.0.0/8\",\n",
" \"152.0.0.0/8\",\n",
" \"153.0.0.0/8\",\n",
" \"154.0.0.0/8\",\n",
" \"155.0.0.0/8\",\n",
" \"156.0.0.0/8\",\n",
" \"157.0.0.0/8\",\n",
" \"158.0.0.0/8\",\n",
" \"159.0.0.0/8\",\n",
" \"160.0.0.0/8\",\n",
" \"161.0.0.0/8\",\n",
" \"162.0.0.0/8\",\n",
" \"163.0.0.0/8\",\n",
" \"164.0.0.0/8\",\n",
" \"165.0.0.0/8\",\n",
" \"166.0.0.0/8\",\n",
" \"167.0.0.0/8\",\n",
" \"168.0.0.0/8\",\n",
" \"169.0.0.0/8\",\n",
" \"170.0.0.0/8\",\n",
" \"171.0.0.0/8\",\n",
" \"172.0.0.0/8\",\n",
" \"173.0.0.0/8\",\n",
" \"174.0.0.0/8\",\n",
" \"175.0.0.0/8\",\n",
" \"176.0.0.0/8\",\n",
" \"177.0.0.0/8\",\n",
" \"178.0.0.0/8\",\n",
" \"179.0.0.0/8\",\n",
" \"180.0.0.0/8\",\n",
" \"181.0.0.0/8\",\n",
" \"182.0.0.0/8\",\n",
" \"183.0.0.0/8\",\n",
" \"184.0.0.0/8\",\n",
" \"185.0.0.0/8\",\n",
" \"186.0.0.0/8\",\n",
" \"187.0.0.0/8\",\n",
" \"188.0.0.0/8\",\n",
" \"189.0.0.0/8\",\n",
" \"190.0.0.0/8\",\n",
" \"191.0.0.0/8\",\n",
" \"192.0.0.0/8\",\n",
" \"193.0.0.0/8\",\n",
" \"194.0.0.0/8\",\n",
" \"195.0.0.0/8\",\n",
" \"196.0.0.0/8\",\n",
" \"197.0.0.0/8\",\n",
" \"198.0.0.0/8\",\n",
" \"199.0.0.0/8\",\n",
" \"200.0.0.0/8\",\n",
" \"201.0.0.0/8\",\n",
" \"202.0.0.0/8\",\n",
" \"203.0.0.0/8\",\n",
" \"204.0.0.0/8\",\n",
" \"205.0.0.0/8\",\n",
" \"206.0.0.0/8\",\n",
" \"207.0.0.0/8\",\n",
" \"208.0.0.0/8\",\n",
" \"209.0.0.0/8\",\n",
" \"210.0.0.0/8\",\n",
" \"211.0.0.0/8\",\n",
" \"212.0.0.0/8\",\n",
" \"213.0.0.0/8\",\n",
" \"214.0.0.0/8\",\n",
" \"215.0.0.0/8\",\n",
" \"216.0.0.0/8\",\n",
" \"217.0.0.0/8\",\n",
" \"218.0.0.0/8\",\n",
" \"219.0.0.0/8\",\n",
" \"220.0.0.0/8\",\n",
" \"221.0.0.0/8\",\n",
" \"222.0.0.0/8\",\n",
" \"223.0.0.0/8\"\n",
"]"
],
"metadata": {
"cellView": "form",
"id": "koPjgI-_w-IH"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title 3. Get Number of host\n",
"def get_host_count(shodan_query):\n",
" try:\n",
" url=SHODAN_COUNT_URL + SHODAN_API_KEY + \"&query=\" + shodan_query\n",
" r=requests.get(url)\n",
"\n",
" if not r.status_code==200:\n",
" return {\"result\": False, \"error\":f\"Status Code {r.status_code} {r.text}\"}\n",
" return {\"result\": True, \"count\": r.json()}\n",
" except Exception as e:\n",
" exc_type, exc_obj, exc_tb = sys.exc_info()\n",
" fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n",
" return {\"result\": False, \"error\": str(e) + \" \" + str(exc_type) + \" \" + fname + \":\" + str(exc_tb.tb_lineno)}\n",
"\n",
"shodan_count_result=get_host_count(shodan_query)\n",
"if not shodan_count_result['result']:\n",
" print(shodan_count_result['error'])\n",
"else:\n",
" number_of_host=shodan_count_result['count']['total']\n",
" print(f\"Number of Host: {number_of_host}\")\n",
"if number_of_host > 1000:\n",
" split_request=True\n",
" print(\"The request will be split.\")\n",
" print(f\"It will take {number_of_host / HOST_PER_MIN} mins\")\n",
"else:\n",
" split_request=False\n",
" print(\"The request will not be split.\")"
],
"metadata": {
"cellView": "form",
"id": "6KE00cqSw-P-"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title 4. Query Shodan\n",
"def query_shodan_one_page(shodan_query, directory, page=1):\n",
" try:\n",
" host_info=[]\n",
" url=SHODAN_SEARCH_URL + SHODAN_API_KEY + \"&query=\" + shodan_query + \"&page=\" + str(page)\n",
" r=requests.get(url)\n",
" if not r.status_code==200:\n",
" return {\"result\": False, \"error\":f\"Status Code {r.status_code} {r.text}\"}\n",
" try:\n",
" api_result=r.json()\n",
" except:\n",
" print(r.text)\n",
" return {\"result\": False, \"error\":\"Can not parse JSON.\"}\n",
" host_info=api_result['matches']\n",
"\n",
" if 'total' in api_result:\n",
" total=api_result['total']\n",
" else:\n",
" total=0\n",
" for h in host_info:\n",
" if not 'ip_str' in h:\n",
" continue\n",
" if \":\" in h['ip_str']:\n",
" continue\n",
" filename=directory + h['ip_str'] + '_' + str(h['port']) + '_' + h['transport'] + '.json'\n",
" if os.path.exists(filename):\n",
" print(f\"[-] Already Exists: {filename}\")\n",
" with open(filename, 'w') as f:\n",
" json.dump(h, f)\n",
" return host_info\n",
" except requests.exceptions.JSONDecodeError:\n",
" exc_type, exc_obj, exc_tb = sys.exc_info()\n",
" fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n",
" return {\"result\": False, \"error\": str(exc_type) + \" \" + fname + \":\" + str(exc_tb.tb_lineno)}\n",
" except Exception as e:\n",
" exc_type, exc_obj, exc_tb = sys.exc_info()\n",
" fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n",
" return {\"result\": False, \"error\": str(e) + \" \" + str(exc_type) + \" \" + fname + \":\" + str(exc_tb.tb_lineno)}\n",
"\n",
"def query_shodan_all_pages(query, directory):\n",
" try_count=0\n",
" page=1\n",
" if not os.path.exists(directory):\n",
" os.mkdir(directory)\n",
" host_count_result=get_host_count(query)\n",
" if not host_count_result['result']:\n",
" print(\"Error:\" + host_count_result['error'])\n",
" exit()\n",
" num_host=host_count_result['count']['total']\n",
" while (page-1) * 100 < num_host:\n",
" print(f\"[+] Query SHODAN (query: {query} count: {page}/{math.ceil(num_host/100)}, error count:{try_count})\")\n",
" host_result=query_shodan_one_page(query, directory, page)\n",
"\n",
" if 'error' in host_result:\n",
" print(f\"[!] Error happening:{host_result['error']}. Retry...\")\n",
" try_count+=1\n",
" if try_count < MAX_ERROR_COUNT:\n",
" time.sleep(3)\n",
" continue\n",
" else:\n",
" input(f\"[?] There is {try_count} errors happening. Do you want to continue?\\n[Ctrl] + [C] to cancel, [Enter] to continue.\")\n",
" try_count=0\n",
" continue\n",
" else:\n",
" page+=1\n",
" time.sleep(1)\n",
"\n",
"\n",
"if not split_request:\n",
" query_shodan_all_pages(shodan_query, output_directory_name)\n",
"else:\n",
" for netblock in GLOBAL_NETBLOCKS:\n",
" query=shodan_query+\" net:\" + netblock\n",
" query_shodan_all_pages(query, output_directory_name)\n",
" time.sleep(1)\n",
"\n"
],
"metadata": {
"cellView": "form",
"id": "Z8zNM7Waw-Ul"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title 5. Compress the result in ZIP\n",
"shutil.make_archive(\"result\", 'zip', output_directory_name)"
],
"metadata": {
"cellView": "form",
"id": "qJnL2GOHHG62"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# 6. Download the zip file\n",
"Now you can download the result.zip from the file on the left pane."
],
"metadata": {
"id": "PeRUWw15ZEBk"
}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment