The Reality of NDN Video Streaming https://yoursunny.com/t/2021/NDN-video-reality/

.gitignore

*.mmdb
*.ndjson
.vscode
__pycache__

README.md

The Reality of NDN Video Streaming

This gist contains code and input data from blog article The Reality of NDN Video Streaming.

To run this script:

1. pipenv install
2. Download MaxMind GeoLite2-ASN.mmdb and GeoLite2-City.mmdb
3. ./process.sh

decode.py

#!/usr/bin/python
import datetime
import ipaddress
import json
import sys
import urllib.parse

import geoip2.database
import ndjson

geoip2asn = geoip2.database.Reader("GeoLite2-ASN.mmdb")
geoip2city = geoip2.database.Reader("GeoLite2-City.mmdb")
writer = ndjson.writer(sys.stdout)
date0 = datetime.datetime(2021, 1, 1, tzinfo=datetime.timezone.utc).timestamp()
date1 = datetime.datetime(2021, 2, 1, tzinfo=datetime.timezone.utc).timestamp()

for j in ndjson.reader(sys.stdin):
    try:
        r = json.loads(urllib.parse.unquote(j["request"]["uri"][1:]))
        if j["ts"] < date0 or j["ts"] >= date1:
            continue
        r["ts"] = j["ts"]

        ip = urllib.parse.urlsplit("//" + j["request"]["remote_addr"]).hostname
        subnet = ipaddress.IPv4Network((ip, 24), strict=False)
        asn = geoip2asn.asn(ip)
        city = geoip2city.city(ip)
        r["ip"] = str(subnet.network_address)
        r["asn"] = asn.autonomous_system_number
        r["location"] = [city.location.longitude, city.location.latitude]
        r["city_accuracy"] = city.city.geoname_id is not None
        r["country"] = city.country.iso_code
        r["continent"] = city.continent.code

        writer.writerow(r)
    except Exception:
        pass

group.py

#!/usr/bin/python
import sys
from collections import defaultdict
from typing import Any, Dict, List, Set

import ndjson
from ndn.encoding.name import Name

from testbed import find_router

Record = Dict[str, Any]


class Video(dict):
    def __init__(self, name: Name.FormalName):
        self.name = name
        self["name"] = Name.to_str(name)
        self["fetch"] = []
        self["playback"] = []

    def add_fetch_record(self, j: Record):
        j["n"] = Name.to_str(Name.from_str(j["n"])[len(self.name):])
        self["fetch"].append(j)

    def add_playback_record(self, j: Record):
        del j["n"]
        self["playback"].append(j)


class Session(dict):
    SESS_KEYS = ["sess", "site", "remote", "ip", "asn",
                 "location", "city_accuracy", "country", "continent"]

    def __init__(self):
        self.fetch: List[Record] = []
        self.playback: List[Record] = []

    def add_record(self, j: Record):
        for KEY in Session.SESS_KEYS:
            self[KEY] = j.pop(KEY, None)
        if j["a"] == "P":
            del j["a"]
            self.playback.append(j)
        else:
            self.fetch.append(j)

    def tag_router(self):
        router = find_router(self["remote"])
        self["router"] = router
        self["router_dist"] = None if router is None \
            else router.geodistance(self["location"])

    def bin_videos(self):
        videos: Dict[bytes, Video] = dict()
        unaccounted: List[Record] = []
        for j in self.playback:
            name = Name.from_str(j["n"])[:-1]
            video = videos.setdefault(Name.to_bytes(name), Video(name))
            video.add_playback_record(j)
        for j in self.fetch:
            name = Name.from_str(j["n"])
            found = False
            for video in videos.values():
                if Name.is_prefix(video.name, name):
                    video.add_fetch_record(j)
                    found = True
                    break
            if not found:
                unaccounted.append(j)
        self["V"] = list(videos.values())
        self["F"] = unaccounted


sessions: defaultdict[str, Session] = defaultdict(lambda: Session())

for j in ndjson.reader(sys.stdin):
    sessions[j["sess"]].add_record(j)

for s in sessions.values():
    s.tag_router()
    s.bin_videos()

ndjson.dump(sessions.values(), sys.stdout)

ndnts-video-beacon.log.xz

Pipfile

[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[pipenv]
allow_prereleases = true

[packages]
geoip2 = "~=4.1.0"
geopy = "~=2.1.0"
ndjson = "~=0.3.1"
python-ndn = "==0.3a1.post1"

[dev-packages]
autopep8 = "*"
isort = "*"

[requires]
python_version = "3.9"

[scripts]
lint = "bash -c \"autopep8 -i *.py && isort *.py\""

Pipfile.lock

{
    "_meta": {
        "hash": {
            "sha256": "22aa8c3e8beee0a97ba5a481d7088abb79b11bf99784eeb0399b0d63a0eb4f9c"
        },
        "pipfile-spec": 6,
        "requires": {
            "python_version": "3.9"
        },
        "sources": [
            {
                "name": "pypi",
                "url": "https://pypi.org/simple",
                "verify_ssl": true
            }
        ]
    },
    "default": {
        "aiohttp": {
            "hashes": [
                "sha256:0b795072bb1bf87b8620120a6373a3c61bfcb8da7e5c2377f4bb23ff4f0b62c9",
                "sha256:0d438c8ca703b1b714e82ed5b7a4412c82577040dadff479c08405e2a715564f",
                "sha256:16a3cb5df5c56f696234ea9e65e227d1ebe9c18aa774d36ff42f532139066a5f",
                "sha256:1edfd82a98c5161497bbb111b2b70c0813102ad7e0aa81cbeb34e64c93863005",
                "sha256:2406dc1dda01c7f6060ab586e4601f18affb7a6b965c50a8c90ff07569cf782a",
                "sha256:2858b2504c8697beb9357be01dc47ef86438cc1cb36ecb6991796d19475faa3e",
                "sha256:2a7b7640167ab536c3cb90cfc3977c7094f1c5890d7eeede8b273c175c3910fd",
                "sha256:3228b7a51e3ed533f5472f54f70fd0b0a64c48dc1649a0f0e809bec312934d7a",
                "sha256:328b552513d4f95b0a2eea4c8573e112866107227661834652a8984766aa7656",
                "sha256:39f4b0a6ae22a1c567cb0630c30dd082481f95c13ca528dc501a7766b9c718c0",
                "sha256:3b0036c978cbcc4a4512278e98e3e6d9e6b834dc973206162eddf98b586ef1c6",
                "sha256:3ea8c252d8df5e9166bcf3d9edced2af132f4ead8ac422eac723c5781063709a",
                "sha256:41608c0acbe0899c852281978492f9ce2c6fbfaf60aff0cefc54a7c4516b822c",
                "sha256:59d11674964b74a81b149d4ceaff2b674b3b0e4d0f10f0be1533e49c4a28408b",
                "sha256:5e479df4b2d0f8f02133b7e4430098699450e1b2a826438af6bec9a400530957",
                "sha256:684850fb1e3e55c9220aad007f8386d8e3e477c4ec9211ae54d968ecdca8c6f9",
                "sha256:6ccc43d68b81c424e46192a778f97da94ee0630337c9bbe5b2ecc9b0c1c59001",
                "sha256:6d42debaf55450643146fabe4b6817bb2a55b23698b0434107e892a43117285e",
                "sha256:710376bf67d8ff4500a31d0c207b8941ff4fba5de6890a701d71680474fe2a60",
                "sha256:756ae7efddd68d4ea7d89c636b703e14a0c686688d42f588b90778a3c2fc0564",
                "sha256:77149002d9386fae303a4a162e6bce75cc2161347ad2ba06c2f0182561875d45",
                "sha256:78e2f18a82b88cbc37d22365cf8d2b879a492faedb3f2975adb4ed8dfe994d3a",
                "sha256:7d9b42127a6c0bdcc25c3dcf252bb3ddc70454fac593b1b6933ae091396deb13",
                "sha256:8389d6044ee4e2037dca83e3f6994738550f6ee8cfb746762283fad9b932868f",
                "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4",
                "sha256:c1e0920909d916d3375c7a1fdb0b1c78e46170e8bb42792312b6eb6676b2f87f",
                "sha256:c68fdf21c6f3573ae19c7ee65f9ff185649a060c9a06535e9c3a0ee0bbac9235",
                "sha256:c733ef3bdcfe52a1a75564389bad4064352274036e7e234730526d155f04d914",
                "sha256:c9c58b0b84055d8bc27b7df5a9d141df4ee6ff59821f922dd73155861282f6a3",
                "sha256:d03abec50df423b026a5aa09656bd9d37f1e6a49271f123f31f9b8aed5dc3ea3",
                "sha256:d2cfac21e31e841d60dc28c0ec7d4ec47a35c608cb8906435d47ef83ffb22150",
                "sha256:dcc119db14757b0c7bce64042158307b9b1c76471e655751a61b57f5a0e4d78e",
                "sha256:df3a7b258cc230a65245167a202dd07320a5af05f3d41da1488ba0fa05bc9347",
                "sha256:df48a623c58180874d7407b4d9ec06a19b84ed47f60a3884345b1a5099c1818b",
                "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7",
                "sha256:f326b3c1bbfda5b9308252ee0dcb30b612ee92b0e105d4abec70335fab5b1245",
                "sha256:f411cb22115cb15452d099fec0ee636b06cf81bfb40ed9c02d30c8dc2bc2e3d1"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==3.7.3"
        },
        "async-timeout": {
            "hashes": [
                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
            ],
            "markers": "python_full_version >= '3.5.3'",
            "version": "==3.0.1"
        },
        "attrs": {
            "hashes": [
                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==20.3.0"
        },
        "certifi": {
            "hashes": [
                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
            ],
            "version": "==2020.12.5"
        },
        "chardet": {
            "hashes": [
                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
            ],
            "version": "==3.0.4"
        },
        "dataclasses": {
            "hashes": [
                "sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f",
                "sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"
            ],
            "version": "==0.6"
        },
        "geographiclib": {
            "hashes": [
                "sha256:12bd46ee7ec25b291ea139b17aa991e7ef373e21abd053949b75c0e9ca55c632",
                "sha256:51cfa698e7183792bce27d8fb63ac8e83689cd8170a730bf35e1a5c5bf8849b9"
            ],
            "version": "==1.50"
        },
        "geoip2": {
            "hashes": [
                "sha256:57d8d15de2527e0697bbef44fc16812bba709f03a07ef99297bd56c1df3b1efd",
                "sha256:707025542ef076bd8fd80e97138bebdb7812527b2a007d141a27ad98b0370fff"
            ],
            "index": "pypi",
            "version": "==4.1.0"
        },
        "geopy": {
            "hashes": [
                "sha256:4db8a2b79a2b3358a7d020ea195be639251a831a1b429c0d1b20c9f00c67c788",
                "sha256:892b219413e7955587b029949af3a1949c6fbac9d5ad17b79d850718f6a9550f"
            ],
            "index": "pypi",
            "version": "==2.1.0"
        },
        "idna": {
            "hashes": [
                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==2.10"
        },
        "maxminddb": {
            "hashes": [
                "sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==2.0.3"
        },
        "multidict": {
            "hashes": [
                "sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a",
                "sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93",
                "sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632",
                "sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656",
                "sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79",
                "sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7",
                "sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d",
                "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5",
                "sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224",
                "sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26",
                "sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea",
                "sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348",
                "sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6",
                "sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76",
                "sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1",
                "sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f",
                "sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952",
                "sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a",
                "sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37",
                "sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9",
                "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359",
                "sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8",
                "sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da",
                "sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3",
                "sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d",
                "sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf",
                "sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841",
                "sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d",
                "sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93",
                "sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f",
                "sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647",
                "sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635",
                "sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456",
                "sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda",
                "sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5",
                "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281",
                "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==5.1.0"
        },
        "ndjson": {
            "hashes": [
                "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410",
                "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6"
            ],
            "index": "pypi",
            "version": "==0.3.1"
        },
        "pycryptodomex": {
            "hashes": [
                "sha256:15c03ffdac17731b126880622823d30d0a3cc7203cd219e6b9814140a44e7fab",
                "sha256:20fb7f4efc494016eab1bc2f555bc0a12dd5ca61f35c95df8061818ffb2c20a3",
                "sha256:28ee3bcb4d609aea3040cad995a8e2c9c6dc57c12183dadd69e53880c35333b9",
                "sha256:305e3c46f20d019cd57543c255e7ba49e432e275d7c0de8913b6dbe57a851bc8",
                "sha256:3547b87b16aad6afb28c9b3a9cd870e11b5e7b5ac649b74265258d96d8de1130",
                "sha256:3642252d7bfc4403a42050e18ba748bedebd5a998a8cba89665a4f42aea4c380",
                "sha256:404faa3e518f8bea516aae2aac47d4d960397199a15b4bd6f66cad97825469a0",
                "sha256:42669638e4f7937b7141044a2fbd1019caca62bd2cdd8b535f731426ab07bde1",
                "sha256:4632d55a140b28e20be3cd7a3057af52fb747298ff0fd3290d4e9f245b5004ba",
                "sha256:4a88c9383d273bdce3afc216020282c9c5c39ec0bd9462b1a206af6afa377cf0",
                "sha256:4ce1fc1e6d2fd2d6dc197607153327989a128c093e0e94dca63408f506622c3e",
                "sha256:55cf4e99b3ba0122dee570dc7661b97bf35c16aab3e2ccb5070709d282a1c7ab",
                "sha256:5e486cab2dfcfaec934dd4f5d5837f4a9428b690f4d92a3b020fd31d1497ca64",
                "sha256:65ec88c8271448d2ea109d35c1f297b09b872c57214ab7e832e413090d3469a9",
                "sha256:6c95a3361ce70068cf69526a58751f73ddac5ba27a3c2379b057efa2f5338c8c",
                "sha256:73240335f4a1baf12880ebac6df66ab4d3a9212db9f3efe809c36a27280d16f8",
                "sha256:7651211e15109ac0058a49159265d9f6e6423c8a81c65434d3c56d708417a05b",
                "sha256:7b5b7c5896f8172ea0beb283f7f9428e0ab88ec248ce0a5b8c98d73e26267d51",
                "sha256:836fe39282e75311ce4c38468be148f7fac0df3d461c5de58c5ff1ddb8966bac",
                "sha256:871852044f55295449fbf225538c2c4118525093c32f0a6c43c91bed0452d7e3",
                "sha256:892e93f3e7e10c751d6c17fa0dc422f7984cfd5eb6690011f9264dc73e2775fc",
                "sha256:934e460c5058346c6f1d62fdf3db5680fbdfbfd212722d24d8277bf47cd9ebdc",
                "sha256:9736f3f3e1761024200637a080a4f922f5298ad5d780e10dbb5634fe8c65b34c",
                "sha256:a1d38a96da57e6103423a446079ead600b450cf0f8ebf56a231895abf77e7ffc",
                "sha256:a385fceaa0cdb97f0098f1c1e9ec0b46cc09186ddf60ec23538e871b1dddb6dc",
                "sha256:a7cf1c14e47027d9fb9d26aa62e5d603994227bd635e58a8df4b1d2d1b6a8ed7",
                "sha256:a9aac1a30b00b5038d3d8e48248f3b58ea15c827b67325c0d18a447552e30fc8",
                "sha256:b696876ee583d15310be57311e90e153a84b7913ac93e6b99675c0c9867926d0",
                "sha256:bef9e9d39393dc7baec39ba4bac6c73826a4db02114cdeade2552a9d6afa16e2",
                "sha256:c885fe4d5f26ce8ca20c97d02e88f5fdd92c01e1cc771ad0951b21e1641faf6d",
                "sha256:d2d1388595cb5d27d9220d5cbaff4f37c6ec696a25882eb06d224d241e6e93fb",
                "sha256:d2e853e0f9535e693fade97768cf7293f3febabecc5feb1e9b2ffdfe1044ab96",
                "sha256:d62fbab185a6b01c5469eda9f0795f3d1a5bba24f5a5813f362e4b73a3c4dc70",
                "sha256:f20a62397e09704049ce9007bea4f6bad965ba9336a760c6f4ef1b4192e12d6d",
                "sha256:f81f7311250d9480e36dec819127897ae772e7e8de07abfabe931b8566770b8e"
            ],
            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==3.9.9"
        },
        "pygtrie": {
            "hashes": [
                "sha256:43205559d28863358dbbf25045029f58e2ab357317a59b11f11ade278ac64692"
            ],
            "version": "==2.4.2"
        },
        "python-ndn": {
            "hashes": [
                "sha256:40cb44c311cfc78a4e7c21f1133a90dd1fdd1a228c2ea57d98e11c560895b515",
                "sha256:c0b846d544aa9e535b55dd24f73e6be5712195e3bfc7dc12741b6900bc7cdc0c"
            ],
            "index": "pypi",
            "version": "==0.3a1.post1"
        },
        "requests": {
            "hashes": [
                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
            "version": "==2.25.1"
        },
        "typing-extensions": {
            "hashes": [
                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
            ],
            "version": "==3.7.4.3"
        },
        "urllib3": {
            "hashes": [
                "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
                "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
            "version": "==1.26.3"
        },
        "yarl": {
            "hashes": [
                "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e",
                "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434",
                "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366",
                "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3",
                "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec",
                "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959",
                "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e",
                "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c",
                "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6",
                "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a",
                "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6",
                "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424",
                "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e",
                "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f",
                "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50",
                "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2",
                "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc",
                "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4",
                "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970",
                "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10",
                "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0",
                "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406",
                "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896",
                "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643",
                "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721",
                "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478",
                "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724",
                "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e",
                "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8",
                "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96",
                "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25",
                "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76",
                "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2",
                "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2",
                "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c",
                "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a",
                "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==1.6.3"
        }
    },
    "develop": {
        "autopep8": {
            "hashes": [
                "sha256:d21d3901cb0da6ebd1e83fc9b0dfbde8b46afc2ede4fe32fbda0c7c6118ca094"
            ],
            "index": "pypi",
            "version": "==1.5.4"
        },
        "isort": {
            "hashes": [
                "sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e",
                "sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc"
            ],
            "index": "pypi",
            "version": "==5.7.0"
        },
        "pycodestyle": {
            "hashes": [
                "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367",
                "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"
            ],
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==2.6.0"
        },
        "toml": {
            "hashes": [
                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
            ],
            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
            "version": "==0.10.2"
        }
    }
}

process.sh

#!/bin/bash
set -eo pipefail

if ! [[ -f testbed.ndjson ]]; then
  pipenv run python testbed.py > testbed.ndjson
fi

# routers by continent
jq -sc 'group_by(.continent)[] | [.[0].continent, (.|length), (.|map(select(.wss))|length)]' testbed.ndjson > testbed-continent.ndjson

if ! [[ -f logs.ndjson ]]; then
  xzcat ndnts-video-beacon.log.xz | pipenv run python decode.py > logs.ndjson
fi

if ! [[ -f sessions.ndjson ]]; then
  pipenv run python group.py < logs.ndjson > sessions.ndjson
fi

# log entries by type
jq -sc 'group_by(.a)[] | [.[0].a, (.|length)]' logs.ndjson > log-type.ndjson

# session video count
jq -sc 'group_by(.V|length)[] | [(.[0].V|length), (.|length)]' sessions.ndjson > session-videos.ndjson

# session location accuracy
jq -sc 'group_by(.city_accuracy)[] | [(.[0].city_accuracy), (.|length)]' sessions.ndjson > session-city-accuracy.ndjson

# session continent and top country
jq -sc 'group_by(.continent)[] | [.[0].continent, (.|length), (.|group_by(.country)|sort_by(.|length)|.[-1] | (.[0].country, (.|length)))]' sessions.ndjson > continent-country.ndjson

# playback duration
jq -sc 'map(.V[].playback | length) | group_by(.)[] | [5*(.[0]), (.|length)]' sessions.ndjson > duration.ndjson

# interval resolution by playback time
jq -sc '. as $a | range(map(.V[].playback | length) | max) | [(. as $i | [null,240,360,480,720,1080] | .[] as $r | [$a[].V[].playback | select((.|length)>$i and .[$i].r==$r)] | length)]' sessions.ndjson > resolution-time.ndjson

# interval resolution by continent
jq -sc '. as $a | map(.continent) | unique[] as $continent | [$continent, ([null,240,360,480,720,1080] | .[] as $r | [$a[] | select(.continent==$continent) | .V[].playback[] | select(.r==$r)] | length)]' sessions.ndjson > resolution-continent.ndjson

# startup latency CDF by continent
jq -sc '. as $a | map(.continent) | unique[] | . as $continent | [$continent, ([$a[] | select(.continent==$continent) | .V[].playback[-1].tl | select(.!=null)] | sort)]' sessions.ndjson > startup-continent.ndjson

# router distance - avg estimated bandwidth - median resolution
jq -c '
def avg: . | map(select(.!=null)) | length as $len | if $len==0 then null else (.|add/$len) end;
def median: . | map(select(.!=null)) | sort | length as $len | if $len==0 then null else .[$len/2 | floor] end;
select((.V|length)>0 and .router_dist!=null) | [.continent, .country, .router_dist, ([.V[].playback[].be] | avg), ([.V[].playback[].r] | median)]' sessions.ndjson | jq -sc '. | sort[]' > router-distance.ndjson

testbed-nodes.json

{
  "UNIVH2C": {
    "ws-tls": false, 
    "neighbors": [
      "WU", 
      "LACL", 
      "UFBA", 
      "PADUA", 
      "COPELABS", 
      "URJC"
    ], 
    "hr_radius": "15.7232", 
    "name": "University Hassan II Casablanca", 
    "ip_addresses": [
      "196.200.164.99"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://196.200.164.99:80/", 
    "prefix": "ndn:/ndn/ma/univh2c", 
    "ndn-up": false, 
    "https": "https://196.200.164.99:443/", 
    "memsize": "unknown", 
    "position": [
      33.5, 
      -7.2
    ], 
    "shortname": "UNIVH2C", 
    "hr_angle": "3.03408"
  }, 
  "AWS-US-W-2": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS US West 2 Region, Oregon", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      44.1, 
      -123.1
    ], 
    "shortname": "AWS-US-W-2"
  }, 
  "MINHO": {
    "ws-tls": false, 
    "neighbors": [
      "AVEIRO", 
      "CORUNA", 
      "COPELABS", 
      "BASEL", 
      "URJC", 
      "PADUA"
    ], 
    "hr_radius": "16.2060", 
    "name": "University oh Minho", 
    "ip_addresses": [
      "193.136.9.206"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://netlab.gcom.di.uminho.pt:80/", 
    "prefix": "ndn:/ndn/pt/uminho", 
    "ndn-up": true, 
    "https": "https://netlab.gcom.di.uminho.pt:443/", 
    "memsize": "4G", 
    "position": [
      41.8, 
      -8.2
    ], 
    "shortname": "MINHO", 
    "hr_angle": "2.62083"
  }, 
  "MUMBAI_AWS": {
    "ws-tls": true, 
    "neighbors": [
      "WU", 
      "AFA", 
      "UUM"
    ], 
    "hr_radius": "12.8934", 
    "name": "MUMBAI gateway at AWS in Mumbai, India", 
    "ip_addresses": [
      "13.126.132.174"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://mumbai.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/aws/mumbai", 
    "ndn-up": true, 
    "https": "https://mumbai.testbed.named-data.net:443/", 
    "memsize": "1G", 
    "position": [
      19.0, 
      73.2
    ], 
    "shortname": "MUMBAI_AWS", 
    "hr_angle": "2.95081"
  }, 
  "MSU": {
    "ws-tls": false, 
    "neighbors": [
      "UUM", 
      "SRRU", 
      "UCLA", 
      "TNO", 
      "ANYANG"
    ], 
    "hr_radius": "19.6533", 
    "name": "Mahasarakham University, Thailand", 
    "ip_addresses": [
      "202.28.34.249"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://msu.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/th/ac/msu", 
    "ndn-up": false, 
    "https": "https://msu.testbed.named-data.net:443/", 
    "memsize": "unknown", 
    "position": [
      17.3, 
      103.7
    ], 
    "shortname": "MSU", 
    "hr_angle": "3.64135"
  }, 
  "AVEIRO": {
    "ws-tls": false, 
    "neighbors": [
      "WU", 
      "URJC", 
      "MINHO", 
      "COPELABS", 
      "AFA"
    ], 
    "hr_radius": "16.2060", 
    "name": "University Aveiro", 
    "ip_addresses": [
      "193.136.92.155"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://selficn.av.it.pt:80/", 
    "prefix": "ndn:/ndn/pt/it/av", 
    "ndn-up": true, 
    "https": "https://selficn.av.it.pt:443/", 
    "memsize": "4G", 
    "position": [
      40.6, 
      -9.0
    ], 
    "shortname": "AVEIRO", 
    "hr_angle": "2.62081"
  }, 
  "BASEL": {
    "ws-tls": true, 
    "neighbors": [
      "BERN", 
      "TNO", 
      "MINHO", 
      "GOETTINGEN", 
      "PADUA", 
      "LIP6", 
      "URJC"
    ], 
    "hr_radius": "13.2717", 
    "name": "University of Basel", 
    "ip_addresses": [
      "192.43.193.111"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://dmi-ndn-testbed1.dmi.unibas.ch:80/", 
    "prefix": "ndn:/ndn/ch/unibas", 
    "ndn-up": true, 
    "https": "https://dmi-ndn-testbed1.dmi.unibas.ch:443/", 
    "memsize": "2G", 
    "position": [
      47.55858, 
      7.5836
    ], 
    "shortname": "BASEL", 
    "hr_angle": "2.41932"
  }, 
  "WU": {
    "ws-tls": true, 
    "neighbors": [
      "MML1", 
      "BERN", 
      "QUB", 
      "WU2", 
      "MUMBAI_AWS", 
      "AVEIRO", 
      "UNIVH2C", 
      "UFBA", 
      "CORUNA", 
      "UCLACS", 
      "UCLA", 
      "URJC", 
      "ARIZONA", 
      "MEMPHIS", 
      "UIUC"
    ], 
    "hr_radius": "19.4268", 
    "name": "Washington University in St. Louis", 
    "ip_addresses": [
      "128.252.153.194"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://wundngw.arl.wustl.edu:80/", 
    "_real_position": [
      38.6490701, 
      -90.3033471
    ], 
    "prefix": "ndn:/ndn/edu/wustl", 
    "ndn-up": true, 
    "https": "https://wundngw.arl.wustl.edu:443/", 
    "memsize": "2G", 
    "position": [
      38.1490701, 
      -90.3033471
    ], 
    "shortname": "WU", 
    "hr_angle": "4.31067"
  }, 
  "NEU": {
    "ws-tls": false, 
    "neighbors": [
      "QUB", 
      "SAVI", 
      "MICHIGAN", 
      "MEMPHIS"
    ], 
    "hr_radius": "19.9001", 
    "name": "Northeastern University", 
    "ip_addresses": [
      "129.10.224.42"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://neu.testbed.named-data.net:80/", 
    "_real_position": [
      42.3398783, 
      -71.0887358
    ], 
    "prefix": "ndn:/ndn/edu/neu", 
    "ndn-up": true, 
    "https": "https://neu.testbed.named-data.net:443/", 
    "memsize": "unknown", 
    "position": [
      42.7, 
      -71.0887358
    ], 
    "shortname": "NEU", 
    "hr_angle": "2.13894"
  }, 
  "AWS-EU-W-2": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS EU West 2 Region, London", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      51.5, 
      0.1
    ], 
    "shortname": "AWS-EU-W-2"
  }, 
  "AWS-EU-W-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS EU West 1 Region, Ireland", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      53.5, 
      -7.0
    ], 
    "shortname": "AWS-EU-W-1"
  }, 
  "AWS-AP-SE-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS AP SouthEast 1 Region, Singapore", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      1.4, 
      103.8
    ], 
    "shortname": "AWS-AP-SE-1"
  }, 
  "AWS-US-W-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS US West 1 Region, N. California", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      41.1, 
      -121.4
    ], 
    "shortname": "AWS-US-W-1"
  }, 
  "UASLP": {
    "ws-tls": false, 
    "neighbors": [
      "UFBA", 
      "ARIZONA", 
      "MEMPHIS"
    ], 
    "hr_radius": "19.0473", 
    "name": "Universidad Aut\u00f3noma de San Luis Potos\u00ed", 
    "ip_addresses": [
      "148.224.92.99"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://uaslp.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/mx/uaslp", 
    "ndn-up": false, 
    "https": "https://uaslp.testbed.named-data.net:443/", 
    "memsize": "8B", 
    "position": [
      22.0, 
      -100.5
    ], 
    "shortname": "UASLP", 
    "hr_angle": "2.96488"
  }, 
  "UIUC": {
    "ws-tls": false, 
    "neighbors": [
      "MICHIGAN", 
      "CSU", 
      "WU", 
      "PADUA"
    ], 
    "hr_radius": "29.2553", 
    "name": "University of Illinois, Urbana-Champaign", 
    "ip_addresses": [
      "72.36.112.82"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndnx.cs.illinois.edu:80/", 
    "_real_position": [
      40.1026039, 
      -88.23171080000002
    ], 
    "prefix": "ndn:/ndn/edu/illinois", 
    "ndn-up": false, 
    "https": "https://ndnx.cs.illinois.edu:443/", 
    "memsize": "8G", 
    "position": [
      40.6026039, 
      -88.23171080000002
    ], 
    "shortname": "UIUC", 
    "hr_angle": "2.98231"
  }, 
  "COPELABS": {
    "ws-tls": false, 
    "neighbors": [
      "UNIVH2C", 
      "AFA", 
      "UFBA", 
      "AVEIRO", 
      "MINHO", 
      "LIP6", 
      "URJC", 
      "PADUA"
    ], 
    "hr_radius": "16.2060", 
    "name": "COPELABS, University Lusofona", 
    "ip_addresses": [
      "193.137.75.171"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://copelabs.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/pt/ulusofona/copelabs", 
    "ndn-up": true, 
    "https": "https://copelabs.testbed.named-data.net:443/", 
    "memsize": "1G", 
    "position": [
      38.6, 
      -9.0
    ], 
    "shortname": "COPELABS", 
    "hr_angle": "2.62082"
  }, 
  "AWS-AP-SE-2": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS AP SouthEast 2 Region, Sydney", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      -33.9, 
      151.0
    ], 
    "shortname": "AWS-AP-SE-2"
  }, 
  "AWS-AP-NE-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS AP NorthEast 1 Region, Tokyo", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "_real_position": [
      35.6, 
      139.9
    ], 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      35.0, 
      140.9
    ], 
    "shortname": "AWS-AP-NE-1"
  }, 
  "PADUA": {
    "ws-tls": false, 
    "neighbors": [
      "UNIVH2C", 
      "AFA", 
      "MINHO", 
      "GOETTINGEN", 
      "COPELABS", 
      "URJC", 
      "UIUC", 
      "BASEL"
    ], 
    "hr_radius": "13.5903", 
    "name": "University of Padua", 
    "ip_addresses": [
      "147.162.114.18"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndnnode.math.unipd.it:80/", 
    "prefix": "ndn:/ndn/it/unipd", 
    "ndn-up": true, 
    "https": "https://ndnnode.math.unipd.it:443/", 
    "memsize": "unknown", 
    "position": [
      45.35858, 
      11.7836
    ], 
    "shortname": "PADUA", 
    "hr_angle": "1.13934"
  }, 
  "LIP6": {
    "ws-tls": true, 
    "neighbors": [
      "MML2", 
      "SAVI", 
      "QUB", 
      "LACL", 
      "BERN", 
      "TNO", 
      "COPELABS", 
      "BASEL", 
      "URJC", 
      "MICHIGAN"
    ], 
    "hr_radius": "29.2553", 
    "name": "University Pierre et Marie Curie, Sorbonne Universities LIP6", 
    "ip_addresses": [
      "132.227.62.163", 
      "2001:660:3302:282c:160::163"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndnhub.ipv6.lip6.fr:80/", 
    "_real_position": [
      48.86, 
      2.34
    ], 
    "prefix": "ndn:/ndn/fr/lip6", 
    "ndn-up": true, 
    "https": "https://ndnhub.ipv6.lip6.fr:443/", 
    "memsize": "4G", 
    "position": [
      48.86, 
      2.34
    ], 
    "shortname": "LIP6", 
    "hr_angle": "2.62295"
  }, 
  "ANYANG": {
    "ws-tls": true, 
    "neighbors": [
      "SRRU", 
      "MSU", 
      "OSAKA", 
      "WASEDA", 
      "UCLA"
    ], 
    "hr_radius": "14.3378", 
    "name": "Anyang University", 
    "ip_addresses": [
      "210.114.89.49"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://anyang.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/kr/anyang", 
    "ndn-up": true, 
    "https": "https://anyang.testbed.named-data.net:443/", 
    "memsize": "48G", 
    "position": [
      37.7, 
      127.0
    ], 
    "shortname": "ANYANG", 
    "hr_angle": "2.99956"
  }, 
  "UFBA": {
    "ws-tls": true, 
    "neighbors": [
      "WU", 
      "UNIVH2C", 
      "UASLP", 
      "MEMPHIS", 
      "COPELABS"
    ], 
    "hr_radius": "29.2553", 
    "name": "Federal University of Bahia, Brazil", 
    "ip_addresses": [
      "200.128.51.61"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ufba.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/br/ufba", 
    "ndn-up": true, 
    "https": "https://ufba.testbed.named-data.net:443/", 
    "memsize": "unknown", 
    "position": [
      -3.7, 
      -49.6
    ], 
    "shortname": "UFBA", 
    "hr_angle": "3.51898"
  }, 
  "URJC": {
    "ws-tls": true, 
    "neighbors": [
      "UNIVH2C", 
      "AVEIRO", 
      "CORUNA", 
      "MINHO", 
      "COPELABS", 
      "PADUA", 
      "BASEL", 
      "WU", 
      "LIP6"
    ], 
    "hr_radius": "13.6888", 
    "name": "Universidad Rey Juan Carlos (King Juan Carlos University)", 
    "ip_addresses": [
      "193.147.79.41"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://insula.gsyc.es:80/", 
    "_real_position": [
      40.3359, 
      -3.8732
    ], 
    "prefix": "ndn:/ndn/es/urjc", 
    "ndn-up": true, 
    "https": "https://insula.gsyc.es:443/", 
    "memsize": "24G", 
    "position": [
      41.3359, 
      -3.8732
    ], 
    "shortname": "URJC", 
    "hr_angle": "1.17169"
  }, 
  "AWS-SA-E-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS SA East 1 Region, Sao Paolo", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      -23.2, 
      -45.0
    ], 
    "shortname": "AWS-SA-E-1"
  }, 
  "LACL": {
    "ws-tls": false, 
    "neighbors": [
      "BERN", 
      "LIP6", 
      "UNIVH2C"
    ], 
    "hr_radius": "12.5144", 
    "name": "Laboratoire D'Algorithmique, Complexite et Logique (LACL), Universite Paris-Est Creteil", 
    "ip_addresses": [
      "193.48.143.172"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndn.lacl.fr:80/", 
    "_real_position": [
      48.5, 
      2.21
    ], 
    "prefix": "ndn:/ndn/fr/lacl", 
    "ndn-up": true, 
    "https": "https://ndn.lacl.fr:443/", 
    "memsize": "unknown", 
    "position": [
      47.5, 
      2.5
    ], 
    "shortname": "LACL", 
    "hr_angle": "2.62296"
  }, 
  "MICHIGAN": {
    "ws-tls": true, 
    "neighbors": [
      "SAVI", 
      "CSU", 
      "UIUC", 
      "MEMPHIS", 
      "NEU", 
      "LIP6"
    ], 
    "hr_radius": "17.7738", 
    "name": "University Michigan", 
    "ip_addresses": [
      "198.111.224.197"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://michigan.testbed.named-data.net:80/", 
    "_real_position": [
      42.1026039, 
      -80.23171080000002
    ], 
    "prefix": "ndn:/ndn/edu/umich", 
    "ndn-up": true, 
    "https": "https://michigan.testbed.named-data.net:443/", 
    "memsize": "4G", 
    "position": [
      42.5026039, 
      -83.4
    ], 
    "shortname": "MICHIGAN", 
    "hr_angle": "2.98018"
  }, 
  "AFA": {
    "ws-tls": true, 
    "neighbors": [
      "AVEIRO", 
      "MUMBAI_AWS", 
      "PADUA", 
      "COPELABS", 
      "BERN"
    ], 
    "hr_radius": "10.3354", 
    "name": "AFA Systems", 
    "ip_addresses": [
      "93.42.157.82"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://93-42-157-82.ip87.fastwebnet.it:80/", 
    "prefix": "ndn:/ndn/it/afasystems", 
    "ndn-up": true, 
    "https": "https://93-42-157-82.ip87.fastwebnet.it:443/", 
    "memsize": "4G", 
    "position": [
      42.1, 
      14.5
    ], 
    "shortname": "AFA", 
    "hr_angle": "1.12640"
  }, 
  "MML1": {
    "ws-tls": true, 
    "neighbors": [
      "WU", 
      "MEMPHIS"
    ], 
    "hr_radius": "14.1225", 
    "name": "Athens University of Economics and Business, Greece", 
    "ip_addresses": [
      "195.251.234.11"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://mmlab-aueb-1.mmlab.edu.gr:80/", 
    "_real_position": [
      37.9942, 
      23.7326
    ], 
    "prefix": "ndn:/ndn/gr/edu/mmlab1", 
    "ndn-up": true, 
    "https": "http://mmlab-aueb-1.mmlab.edu.gr:443/", 
    "memsize": "2G", 
    "position": [
      38.9942, 
      24.7326
    ], 
    "shortname": "MML1", 
    "hr_angle": "3.01596"
  }, 
  "CORUNA": {
    "ws-tls": true, 
    "neighbors": [
      "URJC", 
      "MINHO", 
      "WU"
    ], 
    "hr_radius": "13.6888", 
    "name": "Universidade da Coruna", 
    "ip_addresses": [
      "193.144.50.247"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndn.gtec.udc.es:80/", 
    "prefix": "ndn:/ndn/es/udc", 
    "ndn-up": true, 
    "https": "https://ndn.gtec.udc.es:443/", 
    "memsize": "2G", 
    "position": [
      43.3, 
      -8.2
    ], 
    "shortname": "CORUNA", 
    "hr_angle": "1.17180"
  }, 
  "WASEDA": {
    "ws-tls": false, 
    "neighbors": [
      "OSAKA", 
      "ANYANG", 
      "ARIZONA"
    ], 
    "hr_radius": "19.3583", 
    "name": "Waseda University", 
    "ip_addresses": [
      "133.9.67.98"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndn-tb.nz.comm.waseda.ac.jp:80/", 
    "_real_position": [
      35.7092, 
      139.7193
    ], 
    "prefix": "ndn:/ndn/jp/waseda", 
    "ndn-up": true, 
    "https": "https://ndn-tb.nz.comm.waseda.ac.jp:443/", 
    "memsize": "unknown", 
    "position": [
      35.8092, 
      139.7193
    ], 
    "shortname": "WASEDA", 
    "hr_angle": "2.99935"
  }, 
  "CSU": {
    "ws-tls": false, 
    "neighbors": [
      "ARIZONA", 
      "UCLA", 
      "MICHIGAN", 
      "UIUC"
    ], 
    "hr_radius": "14.1056", 
    "name": "Colorado State University", 
    "ip_addresses": [
      "129.82.138.48"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://mccoy.netsec.colostate.edu:80/", 
    "prefix": "ndn:/ndn/edu/colostate", 
    "ndn-up": true, 
    "https": "https://mccoy.netsec.colostate.edu:443/", 
    "memsize": "4G", 
    "position": [
      40.5747175, 
      -105.0848293
    ], 
    "shortname": "CSU", 
    "hr_angle": "2.99266"
  }, 
  "TNO": {
    "ws-tls": false, 
    "neighbors": [
      "MML2", 
      "QUB", 
      "GOETTINGEN", 
      "LIP6", 
      "BASEL", 
      "MSU"
    ], 
    "hr_radius": "12.0148", 
    "name": "Netherlands Organization for Applied Scientific Research", 
    "ip_addresses": [
      "134.221.116.42"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://tno.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/nl/tno", 
    "ndn-up": true, 
    "https": "https://tno.testbed.named-data.net:443/", 
    "memsize": "4G", 
    "position": [
      52.05858, 
      4.0
    ], 
    "shortname": "TNO", 
    "hr_angle": "2.46031"
  }, 
  "BERN": {
    "ws-tls": false, 
    "neighbors": [
      "WU", 
      "LACL", 
      "LIP6", 
      "BASEL", 
      "AFA"
    ], 
    "hr_radius": "13.2717", 
    "name": "University of Bern", 
    "ip_addresses": [
      "130.92.70.172"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://os-float172.cnds.unibe.ch:80/", 
    "prefix": "ndn:/ndn/ch/unibe", 
    "ndn-up": true, 
    "https": "https://os-float172.cnds.unibe.ch:443/", 
    "memsize": "unknown", 
    "position": [
      46.8, 
      8.5836
    ], 
    "shortname": "BERN", 
    "hr_angle": "2.41933"
  }, 
  "AWS-US-E-2": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS US East 2 Region, Ohio", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "_real_position": [
      40.0, 
      -83.0
    ], 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      40.5, 
      -82.0
    ], 
    "shortname": "AWS-US-E-2"
  }, 
  "AWS-US-E-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS US East 1 Region, N. Virginia", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      38.0, 
      -78.5
    ], 
    "shortname": "AWS-US-E-1"
  }, 
  "AWS-AP-NE-2": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS AP NorthEast 2 Region, Seoul", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "_real_position": [
      37.3, 
      127.0
    ], 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      37.0, 
      128.0
    ], 
    "shortname": "AWS-AP-NE-2"
  }, 
  "MML2": {
    "ws-tls": true, 
    "neighbors": [
      "LIP6", 
      "TNO"
    ], 
    "hr_radius": "14.1226", 
    "name": "Athens University of Economics and Business, Greece", 
    "ip_addresses": [
      "195.251.234.11"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://mmlab-aueb-2.mmlab.edu.gr:80/", 
    "prefix": "ndn:/ndn/gr/edu/mmlab1", 
    "ndn-up": true, 
    "https": "http://mmlab-aueb-2.mmlab.edu.gr:443/", 
    "memsize": "2G", 
    "position": [
      37.9942, 
      23.7326
    ], 
    "shortname": "MML2", 
    "hr_angle": "3.01597"
  }, 
  "AWS-EU-C-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS EU Central 1 Region, Frankfurt", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "_real_position": [
      50.1, 
      8.7
    ], 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      51.0, 
      9.5
    ], 
    "shortname": "AWS-EU-C-1"
  }, 
  "ARIZONA": {
    "ws-tls": true, 
    "neighbors": [
      "UASLP", 
      "UCLACS", 
      "UCLA", 
      "CSU", 
      "MEMPHIS", 
      "WU", 
      "WASEDA", 
      "OSAKA"
    ], 
    "hr_radius": "16.2305", 
    "name": "The University of Arizona", 
    "ip_addresses": [
      "128.196.203.36"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://hobo.cs.arizona.edu:80/", 
    "prefix": "ndn:/ndn/edu/arizona", 
    "ndn-up": true, 
    "https": "https://hobo.cs.arizona.edu:443/", 
    "memsize": "8G", 
    "position": [
      32.2289614, 
      -110.9483226
    ], 
    "shortname": "ARIZONA", 
    "hr_angle": "2.97033"
  }, 
  "AWS-CA-C-1": {
    "ws-tls": false, 
    "neighbors": [], 
    "name": "AWS CA Central 1 Region, Toronto", 
    "ip_addresses": [
      ""
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://0.0.0.0:80/", 
    "_real_position": [
      43.6, 
      -79.4
    ], 
    "ndn-up": false, 
    "https": "https://0.0.0.0:443/", 
    "memsize": "unknown", 
    "position": [
      44.6, 
      -78.4
    ], 
    "shortname": "AWS-CA-W-1"
  }, 
  "SRRU": {
    "ws-tls": false, 
    "neighbors": [
      "MEMPHIS", 
      "UUM", 
      "MSU", 
      "ANYANG", 
      "OSAKA"
    ], 
    "hr_radius": "12.8769", 
    "name": "Surindra Rajabhat University, Thailand", 
    "ip_addresses": [
      "202.29.30.31"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://srru.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/th/ac/srru", 
    "ndn-up": true, 
    "https": "https://srru.testbed.named-data.net:443/", 
    "memsize": "8G", 
    "position": [
      15.3, 
      103.7
    ], 
    "shortname": "SRRU", 
    "hr_angle": "3.63761"
  }, 
  "MEMPHIS": {
    "ws-tls": true, 
    "neighbors": [
      "MML1", 
      "SRRU", 
      "UASLP", 
      "UFBA", 
      "ARIZONA", 
      "WU", 
      "WU2", 
      "MICHIGAN", 
      "NEU"
    ], 
    "hr_radius": "15.2647", 
    "name": "University of Memphis", 
    "ip_addresses": [
      "141.225.11.173"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://titan.cs.memphis.edu:80/", 
    "prefix": "ndn:/ndn/edu/memphis", 
    "ndn-up": true, 
    "https": "https://titan.cs.memphis.edu:443/", 
    "memsize": "4G", 
    "position": [
      35.1186099, 
      -89.93700109999999
    ], 
    "shortname": "MEMPHIS", 
    "hr_angle": "2.98360"
  }, 
  "OSAKA": {
    "ws-tls": false, 
    "neighbors": [
      "UUM", 
      "SRRU", 
      "ANYANG", 
      "GOETTINGEN", 
      "WASEDA", 
      "ARIZONA"
    ], 
    "hr_radius": "19.3573", 
    "name": "Osaka University", 
    "ip_addresses": [
      "133.1.17.51"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndn.ist.osaka-u.ac.jp:80/", 
    "_real_position": [
      34.72, 
      135.5193
    ], 
    "prefix": "ndn:/ndn/jp/ac/osaka-u", 
    "ndn-up": true, 
    "https": "https://ndn.ist.osaka-u.ac.jp:443/", 
    "memsize": "16G", 
    "position": [
      34.3, 
      135.7193
    ], 
    "shortname": "OSAKA", 
    "hr_angle": "3.53687"
  }, 
  "UCLA": {
    "ws-tls": true, 
    "neighbors": [
      "ARIZONA", 
      "WU", 
      "UCLACS", 
      "CSU", 
      "MSU", 
      "ANYANG"
    ], 
    "hr_radius": "16.5866", 
    "name": "University of California, Los Angeles", 
    "ip_addresses": [
      "131.179.196.48"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://suns.cs.ucla.edu:80/", 
    "_real_position": [
      34.0728225, 
      -118.4387398
    ], 
    "prefix": "ndn:/ndn/edu/ucla", 
    "ndn-up": true, 
    "https": "https://suns.cs.ucla.edu:443/", 
    "memsize": "2G", 
    "position": [
      36.5, 
      -130
    ], 
    "shortname": "UCLA", 
    "hr_angle": "2.99911"
  }, 
  "UUM": {
    "ws-tls": true, 
    "neighbors": [
      "MSU", 
      "SRRU", 
      "MUMBAI_AWS", 
      "OSAKA"
    ], 
    "hr_radius": "14.8351", 
    "name": "Universiti Utara Malaysia", 
    "ip_addresses": [
      "203.80.17.88"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://uum.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/my/edu/uum", 
    "ndn-up": true, 
    "https": "https://uum.testbed.named-data.net:443/", 
    "memsize": "16G", 
    "position": [
      6.6, 
      100.3
    ], 
    "shortname": "UUM", 
    "hr_angle": "3.44434"
  }, 
  "QUB": {
    "ws-tls": true, 
    "neighbors": [
      "WU", 
      "NEU", 
      "TNO", 
      "LIP6"
    ], 
    "hr_radius": "17.8582321576696", 
    "name": "Queen's University Belfast, UK", 
    "ip_addresses": [
      "80.76.206.92"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://ndn.qub.ac.uk:80/", 
    "prefix": "ndn:/ndn/uk/ac/qub", 
    "ndn-up": true, 
    "https": "https://ndn.qub.ac.uk:443/", 
    "memsize": "8G", 
    "position": [
      54.5973, 
      -5.9301
    ], 
    "shortname": "QUB", 
    "hr_angle": "2.42743127401714"
  }, 
  "SAVI": {
    "ws-tls": true, 
    "neighbors": [
      "NEU", 
      "MICHIGAN", 
      "LIP6"
    ], 
    "hr_radius": "19.5605", 
    "name": "SAVI, University of Toronto", 
    "ip_addresses": [
      "142.150.208.242"
    ], 
    "backbone": false, 
    "fch-enabled": true, 
    "site": "http://savi.testbed.named-data.net:80/", 
    "prefix": "ndn:/ndn/ca/utoronto", 
    "ndn-up": true, 
    "https": "https://savi.testbed.named-data.net:443/", 
    "memsize": "8G", 
    "position": [
      43.6, 
      -79.4
    ], 
    "shortname": "SAVI", 
    "hr_angle": "4.04925"
  }, 
  "UCLACS": {
    "ws-tls": true, 
    "neighbors": [
      "UCLA", 
      "ARIZONA", 
      "WU"
    ], 
    "hr_radius": "16.5866", 
    "name": "University of California, Los Angeles", 
    "ip_addresses": [
      "131.179.196.46"
    ], 
    "backbone": false, 
    "fch-enabled": false, 
    "site": "http://spurs.cs.ucla.edu:80/", 
    "_real_position": [
      34.0728225, 
      -118.4387398
    ], 
    "prefix": "ndn:/ndn/edu/ucla/cs", 
    "ndn-up": true, 
    "https": "https://spurs.cs.ucla.edu:443/", 
    "memsize": "2G", 
    "position": [
      35.5, 
      -120
    ], 
    "shortname": "UCLACS", 
    "hr_angle": "2.99910"
  }
}

testbed.py

#!/usr/bin/python
import ipaddress
import json
import re
import sys
import urllib.parse
from typing import Any, Dict, List, Optional

import geoip2.database
import ndjson
from geopy.distance import geodesic

Record = Dict[str, Any]

geoip2asn = geoip2.database.Reader("GeoLite2-ASN.mmdb")


class TestbedRouter(dict):
    def __init__(self, j: Record):
        self.router: str = urllib.parse.urlsplit(j["site"]).hostname
        self["host"] = self.router
        self.wss: bool = j["ws-tls"]
        (lon, lat) = j["_real_position"] if "_real_position" in j \
            else j["position"]
        self.lonlat = [lon, lat]
        self["location"] = [lat, lon]
        for ipAddr in j["ip_addresses"]:
            try:
                ip = str(ipaddress.IPv4Address(ipAddr))
                self["ip"] = ip
                self["asn"] = geoip2asn.asn(ip).autonomous_system_number
                break
            except Exception:
                pass

    def geodistance(self, latlon: List[float]) -> float:
        return geodesic(reversed(latlon), self.lonlat).meters


testbed: Dict[str, TestbedRouter] = dict()
with open("testbed-nodes.json") as testbedNodesData:
    for j in json.load(testbedNodesData).values():
        try:
            router = TestbedRouter(j)
            testbed[router.router] = router
        except Exception:
            continue


reWss = re.compile(r"wss:[^)]+")


def find_router(remote: Optional[str]) -> Optional[TestbedRouter]:
    if remote is None:
        return None
    m = reWss.search(remote)
    if m is None:
        return None
    return testbed.get(urllib.parse.urlsplit(m[0]).hostname)


if __name__ == "__main__":
    geoip2city = geoip2.database.Reader("GeoLite2-City.mmdb")
    writer = ndjson.writer(sys.stdout)
    for router in testbed.values():
        router["wss"] = router.wss
        try:
            city = geoip2city.city(router["ip"])
            router["country"] = city.country.iso_code
            router["continent"] = city.continent.code
        except Exception:
            router["country"] = None
            router["continent"] = None
            continue
        writer.writerow(router)