IODAIranData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import csv\n",
    "from datetime import timezone, datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_expression(asn):\n",
    "    return {\n",
    "  \"type\": \"function\",\n",
    "  \"func\": \"removeEmpty\",\n",
    "  \"args\": [\n",
    "    {\n",
    "      \"type\": \"function\",\n",
    "      \"func\": \"group\",\n",
    "      \"args\": [\n",
    "        {\n",
    "          \"type\": \"function\",\n",
    "          \"func\": \"alias\",\n",
    "          \"args\": [\n",
    "            {\n",
    "              \"type\": \"path\",\n",
    "              \"path\": \"bgp.prefix-visibility.asn.{}.v4.visibility_threshold.min_50%_ff_peer_asns.visible_slash24_cnt\".format(asn)\n",
    "            },\n",
    "            {\n",
    "              \"type\": \"constant\",\n",
    "              \"value\": \"BGP (# Visible \\/24s)\"\n",
    "            }\n",
    "          ]\n",
    "        },\n",
    "        {\n",
    "          \"type\": \"function\",\n",
    "          \"func\": \"alias\",\n",
    "          \"args\": [\n",
    "            {\n",
    "              \"type\": \"path\",\n",
    "              \"path\": \"darknet.ucsd-nt.non-erratic.routing.asn.{}.uniq_src_ip\".format(asn)\n",
    "            },\n",
    "            {\n",
    "              \"type\": \"constant\",\n",
    "              \"value\": \"Darknet (# Unique Source IPs)\"\n",
    "            }\n",
    "          ]\n",
    "        },\n",
    "        {\n",
    "          \"type\": \"function\",\n",
    "          \"func\": \"alias\",\n",
    "          \"args\": [\n",
    "            {\n",
    "              \"type\": \"function\",\n",
    "              \"func\": \"sumSeries\",\n",
    "              \"args\": [\n",
    "                {\n",
    "                  \"type\": \"function\",\n",
    "                  \"func\": \"keepLastValue\",\n",
    "                  \"args\": [\n",
    "                    {\n",
    "                      \"type\": \"path\",\n",
    "                      \"path\": \"active.ping-slash24.asn.{}.probers.team-1.caida-sdsc.*.up_slash24_cnt\".format(asn)\n",
    "                    },\n",
    "                    {\n",
    "                      \"type\": \"constant\",\n",
    "                      \"value\": 1\n",
    "                    }\n",
    "                  ]\n",
    "                }\n",
    "              ]\n",
    "            },\n",
    "            {\n",
    "              \"type\": \"constant\",\n",
    "              \"value\": \"Active Probing (# \\/24s Up)\"\n",
    "            }\n",
    "          ]\n",
    "        }\n",
    "      ]\n",
    "    }\n",
    "  ]\n",
    "}\n",
    "\n",
    "def get_chart(asn='12880', since='1573689600', until='1574035200', expression=count_expression):\n",
    "    base_url = 'https://ioda.caida.org/data/ts/json'\n",
    "    params = {\n",
    "        'from': since,\n",
    "        'until': until,\n",
    "        'expression': json.dumps(expression(asn))\n",
    "    }\n",
    "\n",
    "    r = requests.post(base_url, data=params)\n",
    "    j = r.json()\n",
    "    col_name_map = {\n",
    "        'removeEmpty(BGP (# Visible \\\\/24s))': 'bgp',\n",
    "        'removeEmpty(Darknet (# Unique Source IPs))': 'darknet',\n",
    "        'removeEmpty(Active Probing (# \\\\/24s Up))': 'active_probing'\n",
    "    }\n",
    "    cols = {}\n",
    "    for key in j['data']['series'].keys():\n",
    "        col_name = col_name_map[key]\n",
    "        cols[col_name] = j['data']['series'][key]\n",
    "    return cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "asn_list = [\n",
    "    ['44244', 'IranCell'],\n",
    "    ['197207', 'MCCI'],\n",
    "    ['58224', 'IranTelecomCo'],\n",
    "    ['12880', 'ITC'],\n",
    "    ['31549', 'Shatel'],\n",
    "    ['16322', 'ParsOnline'],\n",
    "    ['39501', 'NGSAS'],\n",
    "    ['49100', 'PTE'],\n",
    "    ['60631', 'Pars Parva System']\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading 44244\n"
     ]
    }
   ],
   "source": [
    "with open('20191122-ioda-iran-charts-count.csv', 'w+') as out_file:\n",
    "    writer = csv.DictWriter(out_file, fieldnames=['datasource', 'value', 'asn', 'as_name', 'timestamp'])\n",
    "    \n",
    "    for asnum, asn_name in asn_list:\n",
    "        since = int(datetime(2019, 11, 14, 8, 10, tzinfo=timezone.utc).timestamp())\n",
    "        until = int(datetime(2019, 11, 22, 8, 10, tzinfo=timezone.utc).timestamp())\n",
    "        print('Downloading {}'.format(asnum))\n",
    "        try:\n",
    "            chart = get_chart(asn=asnum, since='{}'.format(since), until='{}'.format(until))\n",
    "        except Exception as exc:\n",
    "            print('Exception {}'.format(exc))\n",
    "            continue\n",
    "\n",
    "        step_range = int((until - since)/300)\n",
    "        for datasource in ['bgp', 'darknet', 'active_probing']:\n",
    "            if datasource not in chart:\n",
    "                print('{} missing!'.format(datasource))\n",
    "                continue\n",
    "            step = chart[datasource]['step']\n",
    "            timestamp = since\n",
    "            idx = 0\n",
    "            while timestamp < until:\n",
    "                writer.writerow({\n",
    "                    'value': chart[datasource]['values'][idx],\n",
    "                    'asn': asnum,\n",
    "                    'as_name': asn_name,\n",
    "                    'datasource': datasource,\n",
    "                    'timestamp': datetime.fromtimestamp(timestamp, timezone.utc)\n",
    "                })\n",
    "                idx += 1\n",
    "                timestamp += step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}