Skip to content

Instantly share code, notes, and snippets.

@0xB10C

0xB10C/pruned-nodes-kits-data.ipynb

Created November 12, 2021 09:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save 0xB10C/c9d715500665df46862e35b0e2b4209d to your computer and use it in GitHub Desktop.
Save 0xB10C/c9d715500665df46862e35b0e2b4209d to your computer and use it in GitHub Desktop.
Download ZIP
Percentage of pruned nods based on https://www.dsn.kastel.kit.edu/bitcoin/ data
Display the source blob
Display the rendered blob
Raw
pruned-nodes-kits-data.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "located-deviation",
"metadata": {},
"outputs": [],
"source": [
"import urllib.request, json \n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "continental-sensitivity",
"metadata": {},
"outputs": [],
"source": [
"NODE_NETWORK = 1\n",
"NODE_WITNESS = 8\n",
"NODE_NETWORK_LIMITED = 1024"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "married-verse",
"metadata": {},
"outputs": [],
"source": [
"# https://www.dsn.kastel.kit.edu/bitcoin/data.html\n",
"# https://www.dsn.kastel.kit.edu/bitcoin/snapshots/\n",
"dataurl = \"https://www.dsn.kastel.kit.edu/bitcoin/snapshots/20211111_123409_dossier.json\""
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "legislative-bathroom",
"metadata": {},
"outputs": [],
"source": [
"with urllib.request.urlopen(dataurl) as url:\n",
" data = json.loads(url.read().decode())"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "twelve-complex",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>lastConnect</th>\n",
" <th>ip</th>\n",
" <th>versionstr</th>\n",
" <th>services</th>\n",
" <th>versionid</th>\n",
" <th>whois</th>\n",
" <th>geo</th>\n",
" <th>latency</th>\n",
" <th>inv</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>654616d13bf625ed</th>\n",
" <td>2021-10-12T16:21:52</td>\n",
" <td>{'version': 6, 'tunnel': 'native'}</td>\n",
" <td>/KITDSN:0.17.0/</td>\n",
" <td>1037</td>\n",
" <td>70015</td>\n",
" <td>{'asn': '34878', 'asn_country_code': 'DE', 'na...</td>\n",
" <td>{'lat': 49.0083, 'long': 8.3979}</td>\n",
" <td>{'btc': {'median': 1, 'count': 30, 'mean': 485...</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>e6cfbe7a19e354a1</th>\n",
" <td>2021-10-12T16:21:52</td>\n",
" <td>{'version': 6, 'tunnel': 'native'}</td>\n",
" <td>/Satoshi:0.21.1/</td>\n",
" <td>1033</td>\n",
" <td>70016</td>\n",
" <td>{'asn': '34878', 'asn_country_code': 'DE', 'na...</td>\n",
" <td>{'lat': 51.0, 'long': 9.0}</td>\n",
" <td>{'btc': {'median': 0, 'count': 30, 'mean': 39,...</td>\n",
" <td>{'invperhour': 13151, 'txperhour': 13090, 'blo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7cdfeeeb64bda830</th>\n",
" <td>2021-10-12T16:21:52</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.18.0/</td>\n",
" <td>1037</td>\n",
" <td>70015</td>\n",
" <td>{'asn': '16509', 'asn_country_code': 'US', 'na...</td>\n",
" <td>{'lat': 50.1153, 'long': 8.6823}</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fd4f572229a3371b</th>\n",
" <td>2021-10-12T16:21:52</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.21.1/</td>\n",
" <td>1033</td>\n",
" <td>70016</td>\n",
" <td>{'asn': '14061', 'asn_country_code': 'US', 'na...</td>\n",
" <td>{'lat': 50.1153, 'long': 8.6823}</td>\n",
" <td>{'icmp': {'median': 1, 'count': 60, 'mean': 1,...</td>\n",
" <td>{'invperhour': 13198, 'txperhour': 13129, 'blo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7ab3cae442e32b03</th>\n",
" <td>2021-10-12T16:21:52</td>\n",
" <td>{'version': 4}</td>\n",
" <td>/Satoshi:0.21.1/</td>\n",
" <td>1033</td>\n",
" <td>70016</td>\n",
" <td>{'asn': '16509', 'asn_country_code': 'US', 'na...</td>\n",
" <td>{'lat': 50.1153, 'long': 8.6823}</td>\n",
" <td>{'icmp': {'median': 2, 'count': 60, 'mean': 2,...</td>\n",
" <td>{'invperhour': 13158, 'txperhour': 13141, 'blo...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" lastConnect ip \\\n",
"654616d13bf625ed 2021-10-12T16:21:52 {'version': 6, 'tunnel': 'native'} \n",
"e6cfbe7a19e354a1 2021-10-12T16:21:52 {'version': 6, 'tunnel': 'native'} \n",
"7cdfeeeb64bda830 2021-10-12T16:21:52 {'version': 4} \n",
"fd4f572229a3371b 2021-10-12T16:21:52 {'version': 4} \n",
"7ab3cae442e32b03 2021-10-12T16:21:52 {'version': 4} \n",
"\n",
" versionstr services versionid \\\n",
"654616d13bf625ed /KITDSN:0.17.0/ 1037 70015 \n",
"e6cfbe7a19e354a1 /Satoshi:0.21.1/ 1033 70016 \n",
"7cdfeeeb64bda830 /Satoshi:0.18.0/ 1037 70015 \n",
"fd4f572229a3371b /Satoshi:0.21.1/ 1033 70016 \n",
"7ab3cae442e32b03 /Satoshi:0.21.1/ 1033 70016 \n",
"\n",
" whois \\\n",
"654616d13bf625ed {'asn': '34878', 'asn_country_code': 'DE', 'na... \n",
"e6cfbe7a19e354a1 {'asn': '34878', 'asn_country_code': 'DE', 'na... \n",
"7cdfeeeb64bda830 {'asn': '16509', 'asn_country_code': 'US', 'na... \n",
"fd4f572229a3371b {'asn': '14061', 'asn_country_code': 'US', 'na... \n",
"7ab3cae442e32b03 {'asn': '16509', 'asn_country_code': 'US', 'na... \n",
"\n",
" geo \\\n",
"654616d13bf625ed {'lat': 49.0083, 'long': 8.3979} \n",
"e6cfbe7a19e354a1 {'lat': 51.0, 'long': 9.0} \n",
"7cdfeeeb64bda830 {'lat': 50.1153, 'long': 8.6823} \n",
"fd4f572229a3371b {'lat': 50.1153, 'long': 8.6823} \n",
"7ab3cae442e32b03 {'lat': 50.1153, 'long': 8.6823} \n",
"\n",
" latency \\\n",
"654616d13bf625ed {'btc': {'median': 1, 'count': 30, 'mean': 485... \n",
"e6cfbe7a19e354a1 {'btc': {'median': 0, 'count': 30, 'mean': 39,... \n",
"7cdfeeeb64bda830 NaN \n",
"fd4f572229a3371b {'icmp': {'median': 1, 'count': 60, 'mean': 1,... \n",
"7ab3cae442e32b03 {'icmp': {'median': 2, 'count': 60, 'mean': 2,... \n",
"\n",
" inv \n",
"654616d13bf625ed NaN \n",
"e6cfbe7a19e354a1 {'invperhour': 13151, 'txperhour': 13090, 'blo... \n",
"7cdfeeeb64bda830 NaN \n",
"fd4f572229a3371b {'invperhour': 13198, 'txperhour': 13129, 'blo... \n",
"7ab3cae442e32b03 {'invperhour': 13158, 'txperhour': 13141, 'blo... "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.DataFrame(data).transpose()\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "signal-official",
"metadata": {},
"outputs": [],
"source": [
"df[\"NODE_NETWORK_LIMITED\"] = df.services.fillna(0).apply(lambda x: bool(int(x) & NODE_NETWORK_LIMITED))\n",
"df[\"NODE_NETWORK\"] = df.services.fillna(0).apply(lambda x: bool(int(x) & NODE_NETWORK))\n",
"df[\"NODE_WITNESS\"] = df.services.fillna(0).apply(lambda x: bool(int(x) & NODE_WITNESS))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "short-needle",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Publicly reachable nodes with NODE_NETWORK, NODE_NETWORK_LIMITED, and NODE_WITNESS set: 76.32%\n"
]
}
],
"source": [
"print(\n",
" \"Publicly reachable nodes with NODE_NETWORK, NODE_NETWORK_LIMITED, and NODE_WITNESS set: %.2f%%\"\n",
" %\n",
" (df[(df.NODE_NETWORK==True) & df.NODE_NETWORK_LIMITED & df.NODE_WITNESS].NODE_WITNESS.count() / df.NODE_WITNESS.count()*100)\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "canadian-offense",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Publicly reachable nodes with NODE_NETWORK _NOT_ set and NODE_NETWORK_LIMITED and NODE_WITNESS set: 12.50%\n"
]
}
],
"source": [
"print(\n",
" \"Publicly reachable nodes with NODE_NETWORK _NOT_ set and NODE_NETWORK_LIMITED and NODE_WITNESS set: %.2f%%\"\n",
" %\n",
" (df[(df.NODE_NETWORK==False) & df.NODE_NETWORK_LIMITED & df.NODE_WITNESS].NODE_WITNESS.count() / df.NODE_WITNESS.count()*100)\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "indirect-appearance",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 for machine learning",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment