Last active
June 1, 2021 12:32
-
-
Save skarfie123/9b08b5041f56526bf7916f20c7177ac6 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/skarfie123/9b08b5041f56526bf7916f20c7177ac6/detectron2-pdm-results.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from collections import OrderedDict\n", | |
| "import numpy as np\n", | |
| "import matplotlib.pyplot as plt\n", | |
| "from pprint import pprint" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "results = {'M_test': OrderedDict([('bbox', {'AP': 42.41234749796167, 'AP50': 57.22575313147572, 'AP75': 45.88676859875574, 'APs': 35.115511551155116, 'APm': 38.500017385636006, 'APl': 53.69180459582039, 'AP-human': 55.16501650165018, 'AP-traffic light': 12.499480717302497, 'AP-street light': 34.24439958370007, 'AP-street sign': 6.732673267326733, 'AP-barrier': 18.019801980198018, 'AP-car': 72.91964780344885, 'AP-pavement': 96.8523102310231, 'AP-drainage': 0.0, 'AP-sidewalk': 74.65003560684809, 'AP-access cover': 50.66831683168317, 'AP-road marking': 44.784139954397716}), ('segm', {'AP': 34.61742155223311, 'AP50': 53.58413492620899, 'AP75': 32.77819042992401, 'APs': \n", | |
| "20.346534653465344, 'APm': 28.858691243116187, 'APl': 51.99380699290413, 'AP-human': 28.613861386138613, 'AP-traffic light': 13.695215675413694, 'AP-street light': 11.80008301582038, 'AP-street sign': 6.732673267326733, 'AP-barrier': 12.871287128712872, 'AP-car': 75.99359180485959, 'AP-pavement': 94.43481848184817, 'AP-drainage': 0.0, 'AP-sidewalk': 62.17657894805166, 'AP-access cover': 52.178217821782184, 'AP-road marking': 22.29530954461025}), ('PDM: Presence', {'traffic light: Precision': 0.5, 'traffic light: Recall': 1.0, 'traffic light: f1': 0.6666666666666666, 'traffic light: Generality': 0.041666666666666664, 'street light: Precision': 0.6470588235294118, 'street light: Recall': 0.7857142857142857, 'street light: f1': 0.7096774193548386, 'street light: Generality': 0.2916666666666667, 'street sign: Precision': 0.3333333333333333, 'street sign: Recall': 0.5, 'street sign: f1': 0.4, 'street sign: Generality': 0.041666666666666664, 'barrier: Precision': 0.3333333333333333, 'barrier: Recall': 0.25, 'barrier: f1': 0.28571428571428575, 'barrier: Generality': 0.08333333333333333, 'pavement: Precision': 1.0, 'pavement: Recall': 1.0, 'pavement: f1': 1.0, 'pavement: Generality': 1.0, 'drainage: Precision': 1, 'drainage: Recall': 0.0, 'drainage: f1': 0.0, 'drainage: Generality': 0.041666666666666664, 'sidewalk: Precision': 1.0, 'sidewalk: Recall': 0.9777777777777777, 'sidewalk: f1': 0.9887640449438202, 'sidewalk: Generality': 0.9375, 'access cover: Precision': 1.0, 'access cover: Recall': 0.625, 'access cover: f1': 0.7692307692307693, 'access cover: Generality': 0.16666666666666666, 'road marking: Precision': 1.0, 'road marking: Recall': 0.9714285714285714, 'road marking: f1': 0.9855072463768115, 'road marking: Generality': 0.7291666666666666, 'Average: Precision': 0.7570806100217865, 'Average: Recall': 0.7344356261022927, 'Average: f1': 0.6561733813652436, 'Average: Generality': 0.3703703703703704}), ('PDM: Detection', {'traffic light: Precision': 0.3076923076923077, 'traffic light: Recall': 1.0, 'traffic light: f1': 0.47058823529411764, 'traffic light: Generality': 0.010723860589812333, 'traffic light: Spatial Error': 0.15909798971150504, 'street light: Precision': 0.5714285714285714, 'street light: Recall': 0.75, 'street light: f1': 0.6486486486486486, 'street light: Generality': 0.04289544235924933, 'street light: Spatial Error': 0.18070830433468266, 'street sign: Precision': 0.16666666666666666, 'street sign: Recall': 0.5, 'street sign: f1': 0.25, 'street sign: Generality': 0.005361930294906166, 'street sign: Spatial Error': 0.10987862603635686, 'barrier: Precision': 0.3333333333333333, 'barrier: Recall': 0.25, 'barrier: f1': 0.28571428571428575, 'barrier: Generality': 0.010723860589812333, 'barrier: Spatial Error': 0.08364656337463523, 'pavement: Precision': 0.94, 'pavement: Recall': 0.9791666666666666, 'pavement: f1': 0.9591836734693877, 'pavement: Generality': 0.128686327077748, 'pavement: Spatial Error': 0.020158809781286376, 'drainage: Precision': 1, 'drainage: Recall': 0.0, 'drainage: f1': 0.0, 'drainage: Generality': 0.00804289544235925, 'drainage: Spatial Error': 0, 'sidewalk: Precision': 0.948051948051948, 'sidewalk: Recall': 0.9125, 'sidewalk: f1': 0.929936305732484, 'sidewalk: Generality': 0.21447721179624665, 'sidewalk: Spatial Error': 0.06570956207552542, 'access cover: Precision': 1.0, 'access cover: Recall': 0.625, 'access cover: f1': 0.7692307692307693, 'access cover: Generality': 0.021447721179624665, 'access cover: Spatial Error': 0.036750374490681095, 'road marking: Precision': 0.7551020408163265, 'road marking: Recall': 0.7956989247311828, 'road marking: f1': 0.7748691099476439, 'road marking: Generality': 0.24932975871313673, 'road marking: Spatial Error': 0.11272066354574838, 'Average: Precision': 0.6691416519987947, 'Average: Recall': 0.6458183990442055, 'Average: f1': 0.5653523364485928, 'Average: Generality': 0.07685433422698838, 'Average: Spatial Error': 0.08540787703893567}), ('PDM: Measurement', {'traffic light: Precision': 0.8061184659456453, 'traffic light: Recall': 0.9209499215626332, 'traffic light: f1': 0.8557355390849818, 'traffic light: Generality': 0.00037558473186728394, 'traffic light: Spatial Error': 0.15909798971150504, 'traffic light: IoU': 0.7532562315559058, 'street light: Precision': 0.8012188347637782, 'street light: Recall': 0.7164972058567992, 'street light: f1': 0.7153261646330721, 'street light: Generality': 0.0007153923128858025, 'street light: Spatial Error': 0.18070830433468266, 'street light: IoU': 0.5826619565541763, 'street sign: Precision': 0.9590395480225988, 'street sign: Recall': 0.6825263863293684, 'street sign: f1': 0.7974943721248898, 'street sign: Generality': 0.00019464940200617284, 'street sign: Spatial Error': 0.10987862603635686, 'street sign: IoU': 0.6631938792121114, 'barrier: Precision': 0.7254082039028276, 'barrier: Recall': 0.9867280606717227, 'barrier: f1': 0.836125774615561, 'barrier: Generality': 0.00011613257137345679, 'barrier: Spatial Error': 0.08364656337463523, 'barrier: IoU': 0.7183987379215145, 'pavement: Precision': 0.9788345091328069, 'pavement: Recall': 0.9855303951607569, 'pavement: f1': 0.9821097434258972, 'pavement: Generality': 0.3109151957947531, 'pavement: Spatial Error': 0.01830984064279615, 'pavement: IoU': 0.9649363271020737, 'drainage: Precision': 1, 'drainage: Recall': 1, 'drainage: f1': 0, 'drainage: Generality': 6.210013181584362e-05, 'drainage: Spatial Error': 0, 'drainage: IoU': 0, 'sidewalk: Precision': 0.9316409802514047, 'sidewalk: Recall': 0.9041074104998731, 'sidewalk: f1': 0.9141082522890481, 'sidewalk: Generality': 0.06176709587191358, 'sidewalk: Spatial Error': 0.046991999690498935, 'sidewalk: IoU': 0.8477528916988122, 'access cover: Precision': 0.927770381515092, 'access cover: Recall': 0.953666587536975, 'access cover: f1': 0.940090549928542, 'access cover: Generality': 0.0012963264371141975, 'access cover: Spatial Error': 0.036750374490681095, 'access cover: IoU': 0.8878351632225684, 'road marking: Precision': 0.8415071231455034, 'road marking: Recall': 0.7408170572575936, 'road marking: f1': 0.7679446067393364, 'road marking: Generality': 0.016699841660236626, 'road marking: Spatial Error': 0.08239740901208674, 'road marking: IoU': 0.6495732408546121, 'Average: Precision': 0.885726449631073, 'Average: Recall': 0.8767581138750802, 'Average: f1': 0.7565483336490365, 'Average: Generality': 0.04357136876821845, 'Average: Spatial Error': 0.07975345636591585, 'Average: IoU': 0.6741787142357527})]), 'G_test': OrderedDict([('bbox', {'AP': 48.64344811344682, 'AP50': 63.17396564995508, 'AP75': 53.01451517573989, 'APs': 8.613861386138613, 'APm': 33.36485947164914, 'APl': 59.57668191082314, 'AP-human': np.nan, 'AP-traffic light': np.nan, 'AP-street light': np.nan, 'AP-street sign': np.nan, 'AP-barrier': np.nan, 'AP-car': 34.05034205437206, 'AP-pavement': 93.1603341727104, 'AP-drainage': 0.0, 'AP-sidewalk': 68.01547780124689, 'AP-access cover': 57.821782178217816, 'AP-road marking': 38.81275247413376}), ('segm', {'AP': 47.42929552155024, 'AP50': 62.510945390433484, 'AP75': 53.543379088383205, 'APs': 6.534653465346535, 'APm': 29.44137303169878, 'APl': 57.952726967582876, 'AP-human': np.nan, 'AP-traffic light': np.nan, 'AP-street light': np.nan, 'AP-street sign': np.nan, 'AP-barrier': np.nan, 'AP-car': 49.790236590982694, 'AP-pavement': 95.18742066514343, 'AP-drainage': 0.0, 'AP-sidewalk': 58.76178899622617, 'AP-access cover': 59.554455445544555, 'AP-road marking': 21.281871431404596}), ('PDM: Presence', {'pavement: Precision': 1.0, 'pavement: Recall': 1.0, 'pavement: f1': 1.0, 'pavement: Generality': 1.0, 'drainage: Precision': 1, 'drainage: Recall': 0.0, 'drainage: f1': 0.0, 'drainage: Generality': 0.041666666666666664, 'sidewalk: Precision': 1.0, 'sidewalk: Recall': 0.9333333333333333, 'sidewalk: f1': 0.9655172413793104, 'sidewalk: Generality': 0.9375, 'access cover: Precision': 1.0, 'access cover: Recall': 0.7142857142857143, 'access cover: f1': 0.8333333333333333, 'access cover: Generality': 0.14583333333333334, 'road marking: Precision': 0.9411764705882353, 'road marking: Recall': 0.9696969696969697, 'road marking: f1': 0.955223880597015, 'road marking: Generality': 0.6875, 'Average: Precision': 0.9882352941176471, 'Average: Recall': 0.7234632034632035, 'Average: f1': 0.7508148910619317, 'Average: Generality': 0.5625}), ('PDM: Detection', {'pavement: Precision': 0.94, 'pavement: Recall': 0.9791666666666666, 'pavement: f1': 0.9591836734693877, 'pavement: Generality': 0.17518248175182483, 'pavement: Spatial Error': 0.03316669399084972, 'drainage: Precision': 1, 'drainage: Recall': 0.0, 'drainage: f1': 0.0, 'drainage: Generality': 0.010948905109489052, 'drainage: Spatial Error': 0, 'sidewalk: Precision': 0.9552238805970149, 'sidewalk: Recall': 0.8421052631578947, 'sidewalk: f1': 0.895104895104895, 'sidewalk: Generality': 0.2773722627737226, 'sidewalk: Spatial Error': 0.053828160970757676, 'access cover: Precision': 1.0, 'access cover: Recall': 0.7142857142857143, 'access cover: f1': 0.8333333333333333, 'access cover: Generality': 0.025547445255474453, 'access cover: Spatial Error': 0.036504976263086505, 'road marking: Precision': 0.61, 'road marking: Recall': 0.7922077922077922, 'road marking: f1': 0.6892655367231638, 'road marking: Generality': 0.28102189781021897, 'road marking: Spatial Error': 0.13359087923372032, 'Average: Precision': 0.9010447761194029, 'Average: Recall': 0.6655530872636135, 'Average: f1': 0.675377487726156, 'Average: Generality': 0.15401459854014599, 'Average: Spatial Error': 0.05141814209168284}), ('PDM: Measurement', {'pavement: Precision': 0.9758184434730911, 'pavement: Recall': 0.9846744595095605, 'pavement: f1': 0.980146754737148, 'pavement: Generality': 0.30820244180812756, 'pavement: Spatial Error': 0.03150564510122175, 'pavement: IoU': 0.9611610597799737, 'drainage: Precision': 1, 'drainage: Recall': 1, 'drainage: f1': 0, 'drainage: Generality': 6.210013181584362e-05, 'drainage: Spatial Error': 0, 'drainage: IoU': 0, 'sidewalk: Precision': 0.9248568234862091, 'sidewalk: Recall': 0.9165071995588335, 'sidewalk: f1': 0.9168350594936366, 'sidewalk: Generality': 0.05954610942322531, 'sidewalk: Spatial Error': 0.0478956167377395, 'sidewalk: IoU': 0.8516038240499255, 'access cover: Precision': 0.9327522944038804, 'access cover: Recall': 0.9525928177800097, 'access cover: f1': 0.9422126762912999, 'access cover: Generality': 0.0011828563046553497, 'access cover: Spatial Error': 0.036504976263086505, 'access cover: IoU': 0.8914637613208534, 'road marking: Precision': 0.8227076213451789, 'road marking: Recall': 0.8085523962099211, 'road marking: f1': 0.8002073854200747, 'road marking: Generality': 0.01469460318608539, 'road marking: Spatial Error': 0.10275892987179101, 'road marking: IoU': 0.6868473145065962, 'Average: Precision': 0.9312270365416719, 'Average: Recall': 0.9324653746116649, 'Average: f1': 0.7278803751884318, 'Average: Generality': 0.0767376221707819, 'Average: Spatial Error': 0.04373303359476775, 'Average: IoU': 0.6782151919314697})]), 'V_test': OrderedDict([('bbox', {'AP': 36.50492178077626, 'AP50': 60.03994947693223, 'AP75': 39.92885472757802, 'APs': 0.0, 'APm': 41.95148182400658, 'APl': 38.03472773925428, 'AP-human': 42.08910891089109, 'AP-traffic light': 31.94719471947195, 'AP-street light': 36.13246324632463, 'AP-street sign': 30.297029702970296, 'AP-barrier': 18.019801980198018, 'AP-car': 60.543932124801536, 'AP-pavement': np.nan, 'AP-drainage': np.nan, 'AP-sidewalk': np.nan, 'AP-access cover': np.nan, 'AP-road marking': np.nan}), ('segm', {'AP': 29.9468409706864, 'AP50': 57.67275543366268, 'AP75': 18.96912775958076, 'APs': 0.0, 'APm': 34.37829579661263, 'APl': 33.815924260099024, 'AP-human': 28.927392739273927, 'AP-traffic light': 34.00990099009901, 'AP-street light': 17.67518894746618, 'AP-street sign': 25.247524752475247, 'AP-barrier': 12.871287128712872, 'AP-car': 60.94975126609118, 'AP-pavement': np.nan, 'AP-drainage': np.nan, 'AP-sidewalk': np.nan, 'AP-access cover': np.nan, 'AP-road marking': np.nan}), ('PDM: Presence', {'traffic light: Precision': 1.0, 'traffic light: Recall': 1.0, 'traffic light: f1': 1.0, 'traffic light: Generality': 0.041666666666666664, 'street light: Precision': 0.8181818181818182, 'street light: Recall': 0.6923076923076923, 'street light: f1': 0.7500000000000001, 'street light: Generality': 0.2708333333333333, 'street sign: Precision': 0.5, 'street sign: Recall': 0.5, 'street sign: f1': 0.5, 'street sign: Generality': 0.041666666666666664, 'barrier: Precision': 0.5, 'barrier: Recall': 0.25, 'barrier: f1': 0.3333333333333333, 'barrier: Generality': 0.08333333333333333, 'Average: Precision': 0.7045454545454546, 'Average: Recall': 0.6105769230769231, 'Average: f1': 0.6458333333333334, 'Average: Generality': 0.109375}), ('PDM: Detection', {'traffic light: Precision': 0.75, 'traffic light: Recall': 0.75, 'traffic light: f1': 0.75, 'traffic light: Generality': 0.03333333333333333, 'traffic light: Spatial Error': 0.11218497089610675, 'street light: Precision': 0.8181818181818182, 'street light: Recall': 0.6, 'street light: f1': 0.6923076923076923, 'street light: Generality': 0.125, 'street light: Spatial Error': 0.16367560737475131, 'street sign: Precision': 0.5, 'street sign: Recall': 0.5, 'street sign: f1': 0.5, 'street sign: Generality': 0.016666666666666666, 'street sign: Spatial Error': 0.13430919236578065, 'barrier: Precision': 0.5, 'barrier: Recall': 0.25, 'barrier: f1': 0.3333333333333333, 'barrier: Generality': 0.03333333333333333, 'barrier: Spatial Error': 0.0852693519556881, 'Average: Precision': 0.6420454545454546, 'Average: Recall': 0.525, 'Average: f1': 0.5689102564102564, 'Average: Generality': 0.05208333333333333, 'Average: Spatial Error': 0.1238597806480817}), ('PDM: Measurement', {'traffic light: Precision': 0.8562826242119765, 'traffic light: Recall': 0.922072913635643, 'traffic light: f1': 0.8860094344142052, 'traffic light: Generality': 0.00037558473186728394, 'traffic light: Spatial Error': 0.11218497089610675, 'traffic light: IoU': 0.7981264097422115, 'street light: Precision': 0.7760067123910911, 'street light: Recall': 0.8134280961505925, 'street light: f1': 0.7866425292088454, 'street light: Generality': 0.0007016581468621399, 'street light: Spatial Error': 0.16367560737475131, 'street light: IoU': 0.6573109067526287, 'street sign: Precision': 0.8685524126455907, 'street sign: Recall': 0.8072164948453608, 'street sign: f1': 0.8367619556505478, 'street sign: Generality': 0.00017366134580761318, 'street sign: Spatial Error': 0.13430919236578065, 'street sign: IoU': 0.7193385392742306, 'barrier: Precision': 0.7260219341974078, 'barrier: Recall': 0.9861863488624052, 'barrier: f1': 0.836338578155507, 'barrier: Generality': 0.00011613257137345679, 'barrier: Spatial Error': 0.0852693519556881, 'barrier: IoU': 0.7187129885511252, 'Average: Precision': 0.8067159208615166, 'Average: Recall': 0.8822259633735005, 'Average: f1': 0.8364381243572764, 'Average: Generality': 0.0003417591989776234, 'Average: Spatial Error': 0.1238597806480817, 'Average: IoU': 0.7233722110800491})])}" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "Constants" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "M, G, V, C = 'M_test', 'G_test', 'V_test', 'C_test'\n", | |
| "Presence, Detection, Measurement = 'PDM: Presence', 'PDM: Detection', 'PDM: Measurement'\n", | |
| "METRIC_PRECISION = \"Precision\"\n", | |
| "METRIC_RECALL = \"Recall\"\n", | |
| "METRIC_F1 = \"f1\"\n", | |
| "METRIC_GENERALITY = \"Generality\"\n", | |
| "METRIC_SPATIAL_ERROR = \"Spatial Error\"\n", | |
| "METRIC_IOU = \"IoU\"\n", | |
| "\n", | |
| "METRICS_P = [METRIC_PRECISION, METRIC_RECALL, METRIC_F1, METRIC_GENERALITY]\n", | |
| "METRICS_D = [\n", | |
| " METRIC_PRECISION,\n", | |
| " METRIC_RECALL,\n", | |
| " METRIC_F1,\n", | |
| " METRIC_GENERALITY,\n", | |
| " METRIC_SPATIAL_ERROR,\n", | |
| "]\n", | |
| "METRICS_M = [\n", | |
| " METRIC_PRECISION,\n", | |
| " METRIC_RECALL,\n", | |
| " METRIC_F1,\n", | |
| " METRIC_GENERALITY,\n", | |
| " METRIC_SPATIAL_ERROR,\n", | |
| " METRIC_IOU,\n", | |
| "]\n", | |
| "# filter\n", | |
| "METRICS_P = [METRIC_PRECISION, METRIC_RECALL, METRIC_F1, METRIC_GENERALITY]\n", | |
| "METRICS_D = [\n", | |
| " METRIC_PRECISION,\n", | |
| " METRIC_RECALL,\n", | |
| " METRIC_F1,\n", | |
| " METRIC_GENERALITY,\n", | |
| " METRIC_SPATIAL_ERROR,\n", | |
| "]\n", | |
| "METRICS_M = [\n", | |
| " METRIC_PRECISION,\n", | |
| " METRIC_RECALL,\n", | |
| " METRIC_F1,\n", | |
| " METRIC_GENERALITY,\n", | |
| " METRIC_SPATIAL_ERROR,\n", | |
| " METRIC_IOU,\n", | |
| "]\n", | |
| "\n", | |
| "ground = {\n", | |
| " \"drainage\",\n", | |
| " \"pavement\",\n", | |
| " \"road marking\",\n", | |
| " \"sidewalk\",\n", | |
| " \"access cover\",\n", | |
| "}\n", | |
| "vertical = {\n", | |
| " \"traffic light\",\n", | |
| " \"street light\",\n", | |
| " \"street sign\",\n", | |
| " \"barrier\",\n", | |
| "}\n", | |
| "merged = ground | vertical\n", | |
| "avg = \"Average\"\n", | |
| "metrics = {Presence: METRICS_P, Detection: METRICS_D, Measurement: METRICS_M}\n", | |
| "metrics_noGen = {}\n", | |
| "for m in metrics:\n", | |
| " metrics_noGen[m] = [i for i in metrics[m] if not i == METRIC_GENERALITY]" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "source": [ | |
| "Reformat" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [ | |
| "outputPrepend" | |
| ] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "print(results.keys())\n", | |
| "import copy\n", | |
| "results2 = copy.deepcopy(results)\n", | |
| "print(results2[M].keys())\n", | |
| "for k in [M, G, V]:\n", | |
| " results2[k].pop(\"bbox\")\n", | |
| " results2[k].pop(\"segm\")\n", | |
| "print(results2[M][Measurement].keys())\n", | |
| "import copy\n", | |
| "results2[C] = copy.deepcopy(results2[G])\n", | |
| "\n", | |
| "for asset in vertical:\n", | |
| " for metric in METRICS_P:\n", | |
| " results2[C][Presence][f\"{asset}: {metric}\"] = results2[V][Presence][f\"{asset}: {metric}\"]\n", | |
| " for metric in METRICS_D:\n", | |
| " results2[C][Detection][f\"{asset}: {metric}\"] = results2[V][Detection][f\"{asset}: {metric}\"]\n", | |
| " for metric in METRICS_M:\n", | |
| " results2[C][Measurement][f\"{asset}: {metric}\"] = results2[V][Measurement][f\"{asset}: {metric}\"]\n", | |
| "\n", | |
| "for metric in METRICS_P:\n", | |
| " results2[C][Presence][f\"{avg}: {metric}\"] = (len(vertical)*results2[V][Presence][f\"{avg}: {metric}\"]+len(ground)*results2[G][Presence][f\"{avg}: {metric}\"])/len(merged)\n", | |
| "for metric in METRICS_D:\n", | |
| " results2[C][Detection][f\"{avg}: {metric}\"] = (len(vertical)*results2[V][Detection][f\"{avg}: {metric}\"]+len(ground)*results2[G][Detection][f\"{avg}: {metric}\"])/len(merged)\n", | |
| "for metric in METRICS_M:\n", | |
| " results2[C][Measurement][f\"{avg}: {metric}\"] = (len(vertical)*results2[V][Measurement][f\"{avg}: {metric}\"]+len(ground)*results2[G][Measurement][f\"{avg}: {metric}\"])/len(merged)\n", | |
| "\n", | |
| "results2.pop(G)\n", | |
| "results2.pop(V)\n", | |
| "\n", | |
| "# pprint(results2)\n", | |
| "results3 = copy.deepcopy(results2)\n", | |
| "for k in results2:\n", | |
| " # results3[k] = {}\n", | |
| " results3[k] = dict(results3[k])\n", | |
| " results3[k][Presence].clear()\n", | |
| " results3[k][Detection].clear()\n", | |
| " results3[k][Measurement].clear()\n", | |
| " for asset in merged:\n", | |
| " results3[k][Presence][asset] = {metric: results2[k][Presence][f\"{asset}: {metric}\"] for metric in METRICS_P}\n", | |
| " results3[k][Detection][asset] = {metric: results2[k][Detection][f\"{asset}: {metric}\"] for metric in METRICS_D}\n", | |
| " results3[k][Measurement][asset] = {metric: results2[k][Measurement][f\"{asset}: {metric}\"] for metric in METRICS_M}\n", | |
| " results3[k][Presence][avg] = {metric: results2[k][Presence][f\"{avg}: {metric}\"] for metric in METRICS_P}\n", | |
| " results3[k][Detection][avg] = {metric: results2[k][Detection][f\"{avg}: {metric}\"] for metric in METRICS_D}\n", | |
| " results3[k][Measurement][avg] = {metric: results2[k][Measurement][f\"{avg}: {metric}\"] for metric in METRICS_M}\n", | |
| "\n", | |
| "# pprint(results3)\n", | |
| "print(len(set(i.split(\":\")[0] for i in results[M][Presence])), set(i.split(\":\")[0] for i in results[M][Presence]))\n", | |
| "print(len(set(i.split(\":\")[0] for i in results[G][Presence])), set(i.split(\":\")[0] for i in results[G][Presence]))\n", | |
| "print(len(set(i.split(\":\")[0] for i in results[V][Presence])), set(i.split(\":\")[0] for i in results[V][Presence]))" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "TSV" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "for k in [M, C]:\n", | |
| " print(k)\n", | |
| " for j in [Presence, Detection, Measurement]:\n", | |
| " print(j)\n", | |
| " print(\"Asset\", end=\"\\t\")\n", | |
| " for metric in metrics[j]:\n", | |
| " print(metric, end=\"\\t\")\n", | |
| " print(\"\")\n", | |
| " for asset in merged:\n", | |
| " print(asset, end=\"\\t\")\n", | |
| " for metric in metrics[j]:\n", | |
| " print(results3[k][j][asset][metric], end=\"\\t\")\n", | |
| " print(\"\")\n", | |
| " print()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "for k in [M, C]:\n", | |
| " print(k)\n", | |
| " s = \",\"\n", | |
| " print(\"Asset\", end=s)\n", | |
| " for j in [Presence, Detection, Measurement]:\n", | |
| " for metric in metrics[j]:\n", | |
| " print(j.split(\": \")[1][0]+\": \"+metric, end=s)\n", | |
| " print(\"\")\n", | |
| " for asset in list(merged)+[avg]:\n", | |
| " print(asset, end=s)\n", | |
| " for j in [Presence, Detection, Measurement]:\n", | |
| " for metric in metrics[j]:\n", | |
| " print(f\"{results3[k][j][asset][metric]:.2g}\", end=s)\n", | |
| " print(\"\")\n", | |
| " print()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "source": [ | |
| "## Graphs" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "from typing import List\n", | |
| "def bargroup(data: List[List[str]], ticks: List[str], labels: List[str], title: str, spacing: float = 0, angle: float = 45):\n", | |
| " x = np.arange(0, len(ticks))*(1+spacing)\n", | |
| " w = 1/len(labels)\n", | |
| " for i,l in enumerate(labels):\n", | |
| " plt.bar(x+i*w, data[i], width=w, label=l)\n", | |
| " plt.xticks(x+0.5*(1-w), ticks, rotation=angle)\n", | |
| "# plt.hlines(1, x[0], x[-1])\n", | |
| " plt.legend()\n", | |
| " plt.title(title)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# graph each metric for Average\n", | |
| "for j in [Presence, Detection, Measurement]:\n", | |
| " bargroup(data=[[results3[l][j][avg][m] for m in metrics[j]] for l in [M,C]], ticks=metrics[j], labels=[M,C], title=f\"{j}, {avg}\", spacing = 0.5)\n", | |
| " name = j.split(\":\")[1].strip()\n", | |
| " plt.savefig(f\"graphs\\\\Average_{name}.png\")\n", | |
| " plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "#graph each metric for each asset\n", | |
| "for j in [Presence, Detection, Measurement]:\n", | |
| " for asset in merged:\n", | |
| " bargroup(data=[[results3[l][j][asset][m] for m in metrics[j]] for l in [M,C]], ticks=metrics[j], labels=[M,C], title=f\"{j}, {asset}\", spacing = 0.5)\n", | |
| " name = j.split(\":\")[1].strip()\n", | |
| " plt.savefig(f\"graphs\\\\asset_{asset}_{name}.png\")\n", | |
| " plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# graph each asset for each metric\n", | |
| "for j in [Presence, Detection, Measurement]:\n", | |
| " for metric in metrics[j]:\n", | |
| " bargroup(data=[[results3[l][j][asset][metric] for asset in merged] for l in [M,C]], ticks=merged, labels=[M,C], title=f\"{j}, {metric}\", spacing = 0.5)\n", | |
| " name = j.split(\":\")[1].strip()\n", | |
| " plt.savefig(f\"graphs\\\\metric_{metric}_{name}.png\")\n", | |
| " plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# graph each asset for each metric, excluding some assets\n", | |
| "merged2 = sorted(set(merged)-set([\"pavement\", \"sidewalk\"]))\n", | |
| "for j in [Presence, Detection, Measurement]:\n", | |
| " for metric in metrics[j]:\n", | |
| " bargroup(data=[[results3[l][j][asset][metric] for asset in merged2] for l in [M,C]], ticks=merged2, labels=[M,C], title=f\"{j}, {metric}\", spacing = 0.5)\n", | |
| " plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "#graph each asset for each metric, normalised by generality\n", | |
| "for j in [Presence, Detection, Measurement]:\n", | |
| " for metric in metrics[j]:\n", | |
| " data = []\n", | |
| " for l in [M,C]:\n", | |
| " d = []\n", | |
| " for asset in merged:\n", | |
| " try:\n", | |
| " value = results3[l][j][asset][metric]/results3[l][j][asset][METRIC_GENERALITY]\n", | |
| " except ZeroDivisionError:\n", | |
| " print(\"Zero Generality for \"+asset)\n", | |
| " value = -1\n", | |
| " d.append(value)\n", | |
| " data.append(d)\n", | |
| " bargroup(data=data, ticks=merged, labels=[M,C], title=f\"{j}, {metric}/Generality\", spacing = 0.5)\n", | |
| " name = j.split(\":\")[1].strip()\n", | |
| " plt.savefig(f\"graphs\\\\metricgen_{metric}_{name}.png\")\n", | |
| " plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "tags": [] | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# relative generality\n", | |
| "for j in [Presence, Detection, Measurement]:\n", | |
| " data = []\n", | |
| " d = []\n", | |
| " for asset in merged:\n", | |
| " try:\n", | |
| " value = results3[M][j][asset][METRIC_GENERALITY]/results3[C][j][asset][METRIC_GENERALITY]\n", | |
| " except ZeroDivisionError:\n", | |
| " print(\"Zero Generality for \"+asset)\n", | |
| " value = -1\n", | |
| " d.append(value)\n", | |
| " data.append(d)\n", | |
| " bargroup(data=data, ticks=merged, labels=[M+\"/\"+C], title=f\"{j}, Relative Generality\", spacing = 0.5)\n", | |
| " plt.axhline(1, color=\"k\")\n", | |
| " name = j.split(\":\")[1].strip()\n", | |
| " plt.savefig(f\"graphs\\\\relgen_{name}.png\")\n", | |
| " plt.show()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "counts={ \"G_test\": {\"images\":48, \"total\":274, 'human': 0, 'traffic light': 0, 'street light': 0, 'street sign': 0, 'barrier': 0, 'car': 63, 'pavement': 48, 'drainage': 3, 'sidewalk': 76, 'access cover': 7, 'road marking': 77},\n", | |
| "\"G_train\": {\"images\":219, \"total\":119, 'human': 8, 'traffic light': 0, 'street light': 0, 'street sign': 0, 'barrier': 0, 'car': 274, 'pavement': 222, 'drainage': 9, 'sidewalk': 322, 'access cover': 15, 'road marking': 341},\n", | |
| "\"G_val\": {\"images\":47, \"total\":248, 'human': 3, 'traffic light': 0, 'street light': 0, 'street sign': 0, 'barrier': 0, 'car': 60, 'pavement': 47, 'drainage': 2, 'sidewalk': 66, 'access cover': 8, 'road marking': 62},\n", | |
| "\"M_test\": {\"images\":48, \"total\":373, 'human': 10, 'traffic light': 4, 'street light': 16, 'street sign': 2, 'barrier': 4, 'car': 105, 'pavement': 48, 'drainage': 3, 'sidewalk': 80, 'access cover': 8, 'road marking': 93},\n", | |
| "\"M_train\": {\"images\":219, \"total\":157, 'human': 40, 'traffic light': 14, 'street light': 86, 'street sign': 18, 'barrier': 10, 'car': 423, 'pavement': 221, 'drainage': 9, 'sidewalk': 336, 'access cover': 16, 'road marking': 404},\n", | |
| "\"M_val\": {\"images\":47, \"total\":327, 'human': 8, 'traffic light': 5, 'street light': 16, 'street sign': 7, 'barrier': 4, 'car': 82, 'pavement': 47, 'drainage': 4, 'sidewalk': 68, 'access cover': 9, 'road marking': 77},\n", | |
| "\"M_valtest\": {\"images\":95, \"total\":700, 'human': 18, 'traffic light': 9, 'street light': 32, 'street sign': 9, 'barrier': 8, 'car': 187, 'pavement': 95, 'drainage': 7, 'sidewalk': 148, 'access cover': 17, 'road marking': 170},\n", | |
| "\"V_test\": {\"images\":48, \"total\":120, 'human': 8, 'traffic light': 4, 'street light': 15, 'street sign': 2, 'barrier': 4, 'car': 87, 'pavement': 0, 'drainage': 0, 'sidewalk': 0, 'access cover': 0, 'road marking': 0},\n", | |
| "\"V_train\": {\"images\":219, \"total\":533, 'human': 38, 'traffic light': 12, 'street light': 76, 'street sign': 17, 'barrier': 8, 'car': 382, 'pavement': 0, 'drainage': 0, 'sidewalk': 0, 'access cover': 0, 'road marking': 0},\n", | |
| "\"V_val\": {\"images\":47, \"total\":116, 'human': 6, 'traffic light': 5, 'street light': 15, 'street sign': 8, 'barrier': 4, 'car': 78, 'pavement': 0, 'drainage': 0, 'sidewalk': 0, 'access cover': 0, 'road marking': 0},\n", | |
| " }" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "print(\"Dataset\", *(counts[\"G_train\"].keys()), sep=\"\\t\")\n", | |
| "for i in counts:\n", | |
| " print(i, *(counts[i].values()), sep=\"\\t\")" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.9.3" | |
| }, | |
| "metadata": { | |
| "interpreter": { | |
| "hash": "ca9d65acf3933d496d8b24c4ca67d14f59eb8c379ac1b81aa351e01d1f505e57" | |
| } | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 4 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment