LK perf status

AlgoMetrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7c1ab24f",
   "metadata": {},
   "source": [
    "# Algorithm Metrics\n",
    "\n",
    "This notebook shows algorithm metrics over the release branches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d00fcf6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:28.369453Z",
     "iopub.status.busy": "2021-06-18T16:47:28.369453Z",
     "iopub.status.idle": "2021-06-18T16:47:28.377905Z",
     "shell.execute_reply": "2021-06-18T16:47:28.377905Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import subprocess\n",
    "from packaging.version import parse as parse_version\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a27c08ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:28.380905Z",
     "iopub.status.busy": "2021-06-18T16:47:28.380905Z",
     "iopub.status.idle": "2021-06-18T16:47:29.621712Z",
     "shell.execute_reply": "2021-06-18T16:47:29.621712Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac3f1886",
   "metadata": {},
   "source": [
    "## Load DVC metrics\n",
    "\n",
    "Let's load all the DVC metrics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3f6c21d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:29.625712Z",
     "iopub.status.busy": "2021-06-18T16:47:29.624711Z",
     "iopub.status.idle": "2021-06-18T16:47:31.814724Z",
     "shell.execute_reply": "2021-06-18T16:47:31.814724Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['main', 'versions/0.10', 'versions/0.11', 'versions/0.12', 'versions/0.9']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proc = subprocess.run(['dvc', 'metrics', 'show', '-a', '--show-json'], stdout=subprocess.PIPE, text=True)\n",
    "metrics = json.loads(proc.stdout)\n",
    "list(metrics.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "abeef813",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:31.818693Z",
     "iopub.status.busy": "2021-06-18T16:47:31.818693Z",
     "iopub.status.idle": "2021-06-18T16:47:31.830693Z",
     "shell.execute_reply": "2021-06-18T16:47:31.829693Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0.9', '0.10', '0.11', '0.12', 'main']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "versions = [v.replace(r'versions/', '') for v in metrics.keys() if v != 'main']\n",
    "versions.sort(key=parse_version)\n",
    "versions.append('main')\n",
    "versions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d35f01d7",
   "metadata": {},
   "source": [
    "Now let's define a function that can traverses the metric structure and yields individual rows for a table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f0cac6eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:31.834693Z",
     "iopub.status.busy": "2021-06-18T16:47:31.834693Z",
     "iopub.status.idle": "2021-06-18T16:47:31.845693Z",
     "shell.execute_reply": "2021-06-18T16:47:31.844693Z"
    }
   },
   "outputs": [],
   "source": [
    "_mfn_re = re.compile(r'^runs/(\\w+)-(.*)\\.json$')\n",
    "def metric_rows(metrics):\n",
    "    for v, files in metrics.items():\n",
    "        v = v.replace('versions/', '')\n",
    "        for fn, vals in files.items():\n",
    "            fn = fn.replace('\\\\', '/')\n",
    "            m = _mfn_re.match(fn)\n",
    "            if m:\n",
    "                data = m.group(1)\n",
    "                algo = m.group(2)\n",
    "                row = {\n",
    "                    'version': v,\n",
    "                    'algo': algo,\n",
    "                    'data': data,\n",
    "                }\n",
    "                row.update(vals)\n",
    "                yield row"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bfd80c0",
   "metadata": {},
   "source": [
    "And compute a full data frame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d14ab726",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:31.858697Z",
     "iopub.status.busy": "2021-06-18T16:47:31.858697Z",
     "iopub.status.idle": "2021-06-18T16:47:31.891692Z",
     "shell.execute_reply": "2021-06-18T16:47:31.890708Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>version</th>\n",
       "      <th>algo</th>\n",
       "      <th>data</th>\n",
       "      <th>nDCG</th>\n",
       "      <th>GRMSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>main</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.076627</td>\n",
       "      <td>0.931476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>main</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.058912</td>\n",
       "      <td>0.886666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.041722</td>\n",
       "      <td>0.857724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>main</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.032337</td>\n",
       "      <td>0.856089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>main</td>\n",
       "      <td>II</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.056826</td>\n",
       "      <td>0.926470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>main</td>\n",
       "      <td>II</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.043657</td>\n",
       "      <td>0.879347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>main</td>\n",
       "      <td>II</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.013582</td>\n",
       "      <td>0.847025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>main</td>\n",
       "      <td>II</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.001916</td>\n",
       "      <td>0.843537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>main</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.036631</td>\n",
       "      <td>0.948779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>main</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.035071</td>\n",
       "      <td>0.917861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>main</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.010922</td>\n",
       "      <td>0.876875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.10</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.077229</td>\n",
       "      <td>0.931957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.10</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.058967</td>\n",
       "      <td>0.886813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.10</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.041757</td>\n",
       "      <td>0.857733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.10</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.032364</td>\n",
       "      <td>0.856086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.10</td>\n",
       "      <td>II</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.056826</td>\n",
       "      <td>0.926470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.10</td>\n",
       "      <td>II</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.043657</td>\n",
       "      <td>0.879347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.10</td>\n",
       "      <td>II</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.013582</td>\n",
       "      <td>0.847025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.10</td>\n",
       "      <td>II</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.001916</td>\n",
       "      <td>0.843537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.10</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.036631</td>\n",
       "      <td>0.948779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.10</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.035071</td>\n",
       "      <td>0.917861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.10</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.010906</td>\n",
       "      <td>0.876329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.11</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.074528</td>\n",
       "      <td>0.993842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.11</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.040611</td>\n",
       "      <td>0.958210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.11</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.018750</td>\n",
       "      <td>0.940330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.11</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.010868</td>\n",
       "      <td>0.954600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.11</td>\n",
       "      <td>II</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.056826</td>\n",
       "      <td>0.926470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.11</td>\n",
       "      <td>II</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.043657</td>\n",
       "      <td>0.879347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.11</td>\n",
       "      <td>II</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.013582</td>\n",
       "      <td>0.847025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.11</td>\n",
       "      <td>II</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.001916</td>\n",
       "      <td>0.843537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.11</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.036631</td>\n",
       "      <td>0.948779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.11</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.035071</td>\n",
       "      <td>0.917861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.11</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.010906</td>\n",
       "      <td>0.876329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0.12</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.073727</td>\n",
       "      <td>0.993145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>0.12</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.040674</td>\n",
       "      <td>0.958002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.12</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.018765</td>\n",
       "      <td>0.940325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.12</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.010870</td>\n",
       "      <td>0.954604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.12</td>\n",
       "      <td>II</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.056826</td>\n",
       "      <td>0.926470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.12</td>\n",
       "      <td>II</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.043657</td>\n",
       "      <td>0.879347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.12</td>\n",
       "      <td>II</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.013582</td>\n",
       "      <td>0.847025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.12</td>\n",
       "      <td>II</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.001916</td>\n",
       "      <td>0.843537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.9</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.075863</td>\n",
       "      <td>0.931913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.9</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.058803</td>\n",
       "      <td>0.886654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.9</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.041716</td>\n",
       "      <td>0.857729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.9</td>\n",
       "      <td>ALS</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.032356</td>\n",
       "      <td>0.856088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.9</td>\n",
       "      <td>II</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.056826</td>\n",
       "      <td>0.926470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.9</td>\n",
       "      <td>II</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.043657</td>\n",
       "      <td>0.879347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>0.9</td>\n",
       "      <td>II</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.013582</td>\n",
       "      <td>0.847025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.9</td>\n",
       "      <td>II</td>\n",
       "      <td>ml20m</td>\n",
       "      <td>0.001916</td>\n",
       "      <td>0.843537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.9</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml100k</td>\n",
       "      <td>0.036631</td>\n",
       "      <td>0.948779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.9</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml1m</td>\n",
       "      <td>0.035071</td>\n",
       "      <td>0.917861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>0.9</td>\n",
       "      <td>UU</td>\n",
       "      <td>ml10m</td>\n",
       "      <td>0.010906</td>\n",
       "      <td>0.876329</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   version algo    data      nDCG     GRMSE\n",
       "0     main  ALS  ml100k  0.076627  0.931476\n",
       "1     main  ALS    ml1m  0.058912  0.886666\n",
       "2     main  ALS   ml10m  0.041722  0.857724\n",
       "3     main  ALS   ml20m  0.032337  0.856089\n",
       "4     main   II  ml100k  0.056826  0.926470\n",
       "5     main   II    ml1m  0.043657  0.879347\n",
       "6     main   II   ml10m  0.013582  0.847025\n",
       "7     main   II   ml20m  0.001916  0.843537\n",
       "8     main   UU  ml100k  0.036631  0.948779\n",
       "9     main   UU    ml1m  0.035071  0.917861\n",
       "10    main   UU   ml10m  0.010922  0.876875\n",
       "11    0.10  ALS  ml100k  0.077229  0.931957\n",
       "12    0.10  ALS    ml1m  0.058967  0.886813\n",
       "13    0.10  ALS   ml10m  0.041757  0.857733\n",
       "14    0.10  ALS   ml20m  0.032364  0.856086\n",
       "15    0.10   II  ml100k  0.056826  0.926470\n",
       "16    0.10   II    ml1m  0.043657  0.879347\n",
       "17    0.10   II   ml10m  0.013582  0.847025\n",
       "18    0.10   II   ml20m  0.001916  0.843537\n",
       "19    0.10   UU  ml100k  0.036631  0.948779\n",
       "20    0.10   UU    ml1m  0.035071  0.917861\n",
       "21    0.10   UU   ml10m  0.010906  0.876329\n",
       "22    0.11  ALS  ml100k  0.074528  0.993842\n",
       "23    0.11  ALS    ml1m  0.040611  0.958210\n",
       "24    0.11  ALS   ml10m  0.018750  0.940330\n",
       "25    0.11  ALS   ml20m  0.010868  0.954600\n",
       "26    0.11   II  ml100k  0.056826  0.926470\n",
       "27    0.11   II    ml1m  0.043657  0.879347\n",
       "28    0.11   II   ml10m  0.013582  0.847025\n",
       "29    0.11   II   ml20m  0.001916  0.843537\n",
       "30    0.11   UU  ml100k  0.036631  0.948779\n",
       "31    0.11   UU    ml1m  0.035071  0.917861\n",
       "32    0.11   UU   ml10m  0.010906  0.876329\n",
       "33    0.12  ALS  ml100k  0.073727  0.993145\n",
       "34    0.12  ALS    ml1m  0.040674  0.958002\n",
       "35    0.12  ALS   ml10m  0.018765  0.940325\n",
       "36    0.12  ALS   ml20m  0.010870  0.954604\n",
       "37    0.12   II  ml100k  0.056826  0.926470\n",
       "38    0.12   II    ml1m  0.043657  0.879347\n",
       "39    0.12   II   ml10m  0.013582  0.847025\n",
       "40    0.12   II   ml20m  0.001916  0.843537\n",
       "41     0.9  ALS  ml100k  0.075863  0.931913\n",
       "42     0.9  ALS    ml1m  0.058803  0.886654\n",
       "43     0.9  ALS   ml10m  0.041716  0.857729\n",
       "44     0.9  ALS   ml20m  0.032356  0.856088\n",
       "45     0.9   II  ml100k  0.056826  0.926470\n",
       "46     0.9   II    ml1m  0.043657  0.879347\n",
       "47     0.9   II   ml10m  0.013582  0.847025\n",
       "48     0.9   II   ml20m  0.001916  0.843537\n",
       "49     0.9   UU  ml100k  0.036631  0.948779\n",
       "50     0.9   UU    ml1m  0.035071  0.917861\n",
       "51     0.9   UU   ml10m  0.010906  0.876329"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf = pd.DataFrame.from_records(metric_rows(metrics))\n",
    "mdf = mdf.astype({'version': 'category'})\n",
    "mdf['version'].cat.reorder_categories(versions, inplace=True)\n",
    "mdf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c34ac96",
   "metadata": {},
   "source": [
    "## ALS Results\n",
    "\n",
    "Let's first look at biased MF from ALS:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ceba2a39",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:31.912690Z",
     "iopub.status.busy": "2021-06-18T16:47:31.900700Z",
     "iopub.status.idle": "2021-06-18T16:47:32.092690Z",
     "shell.execute_reply": "2021-06-18T16:47:32.093688Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='version', ylabel='GRMSE'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEJCAYAAACOr7BbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABTiElEQVR4nO3dd3gU1dfA8e9JIyG0VFrohN4JXUV/iIAi0hSwoCBNQREsKL6U2BtNEZAiTRSliIgNUbHREiB0kA4BJBVCgPT7/jEbXMKm72Z3k/t5njzu7tyZObuSnJ2ZM+eKUgpN0zRNy8rF3gFomqZpjkknCE3TNM0inSA0TdM0i3SC0DRN0yzSCULTNE2zSCcITdM0zSKbJQgR+VREokRkfzbLRUQ+FJFjIrJXRFqZLesuIkdMy162VYyapmla9mx5BLEE6J7D8h5AsOlnBDAXQERcgY9NyxsBg0SkkQ3j1DRN0yxws9WGlVJ/iEjNHIY8ACxTxp1620SkgohUBmoCx5RSJwBEZKVp7MHc9unv769q1sxpl5qmaZq5nTt3xiilAiwts1mCyIOqwFmz55Gm1yy93i67jYjICIwjEKpXr054eLj1I9U0TSumROR0dsvseZFaLLymcnjdIqXUfKVUiFIqJCDAYhLUNE3TCsCeRxCRQDWz50HAecAjm9c1TdO0ImTPI4j1wGBTNVN74LJS6gIQBgSLSC0R8QAGmsZqmqZpRchmRxAi8gVwJ+AvIpHAFMAdQCk1D/geuBc4BlwDhpiWpYnIGOAnwBX4VCl1oKBxpKamEhkZSVJSUiHeTfHl6elJUFAQ7u7u9g5F0zQHY8sqpkG5LFfA6GyWfY+RQAotMjKSsmXLUrNmTUQsXd4ouZRSxMbGEhkZSa1atewdjqZpDqbY30mdlJSEn5+fTg4WiAh+fn766ErTNIuKfYIAdHLIgf5sNE3Ljj2rmDStRNhyPIZtJ+JwFcHNVXARwdUFXF1ccBVwdRHjceZrLuAigpv54xvrmX4svWZ6PfNx5no3veYiuLn8t56bi+gvCVq2dIKwkalTp1KmTBleeOEFi8vXrVtHvXr1aNRIdxEpzrYci2HwpztIy3DcqX1FuClp5DnZZJO4btqGhdeM5GckK1cXcHNxybItaFvLj8719H1N9qYThJ2sW7eOnj176gRRjB2LusLIz3ZSy9+b1U91xNvDlXSlSM8wfjIyIC0jg3T13+OMDExjMkjP4MZY8/XMX8vIUKRlbk8ZjzOyjLG0nvnzrOulmV7LaR3z9W48TlekZWSQnGa+Dqb3Yh6z6X1nYFovgwz133tNy8hg7ubjLHq8DXc1CLT3/8YSTScIK3rzzTdZtmwZ1apVIyAggNatW7NgwQLmz59PSkoKdevWZfny5URERLB+/Xp+//133njjDdasWcOvv/56y7jSpUvb+y1pBRSTmMyQJWGUcnPh0yfaUN7LKCPWv3C5u5qcxoD5Wxnz+S6+GtWBxlXK2zukEqtEXKQuCjt37mTlypXs3r2btWvXEhYWBkDfvn0JCwtjz549NGzYkEWLFtGxY0d69erF+++/T0REBHXq1LE4TnNOSanpDF8WTlRCMgsfb0M1X53o88O7lBuLHm9DOS93nlwSzr+XdZWdvegEYSV//vknffr0oXTp0pQrV45evXoBsH//fm6//XaaNm3KihUrOHDA8j1/eR2nObaMDMXzq/aw+8wlZg5oQYtqFewdklOqWM6TT59ow5WkVIYuCeNqcpq9QyqRdIKwIkvVIE888QSzZ89m3759TJkyJdt7DvI6TnNs034+wnd7L/Byjwb0aFrZ3uE4tYaVy/HxI604cvEKz3yxm3QHvtBfXOkEYSV33HEHX3/9NdevX+fKlSt8++23AFy5coXKlSuTmprKihUrbowvW7YsV65cufE8u3Ga8/gq/Cwf/3acQW2rMfKO2vYOp1i4s34grz3QmF8PR/H6hlynhNGsTF8zs5JWrVoxYMAAWrRoQY0aNbj99tsBeP3112nXrh01atSgadOmN5LCwIEDGT58OB9++CGrV6/OdpzmHLYci2Hi2n3cHuzPaw800fcWWNEj7WpwOvYa8/84QXXf0gy9TbeFKSpitEQqHkJCQlTWCYMOHTpEw4YN7RSRc9CfUeEci7pC3zlbqFjOkzVPd6Scp258aG0ZGYqnV+zip4P/8smjrbmncSV7h1RsiMhOpVSIpWX6FJOmFUJmOauHqZxVJwfbcHERZgxoQbOq5Rm7MoJ9kZftHVKJoBOEphVQUmo6I0zlrAsGh+hyVhvz8nBlweMh+Hp7MHRpGOcuXbd3SMWeThCaVgAZGYoXVu1hl6mctWV1H3uHVCIElvVk8ZA2JKWk8+SSMK4kpdo7pGJNJwhNK4BpPx9hgy5ntYt6Fcsy99HWHItKZPTnu0lLz7B3SMWWThCalk+Z5awD2+hyVnu5LdifN/s04Y9/opm8/gDFqdjGkegyV03Lh8xy1tvq+vN6b13Oak8D2lTndOw15mw+Ti0/b4brZG11+gjCASxZsoQxY8YA8Mcff9CqVSvc3NxYvXr1TeOWLl1KcHAwwcHBLF269MbrJ0+epF27dgQHBzNgwABSUlIAo+X4Bx98UHRvpJg7FpXIKFN31jmPtsLdVf/62NsL99TnvmaVeeuHQ/yw74K9wyl29L9wB1O9enWWLFnCww8/fNPrcXFxhIaGsn37dnbs2EFoaCjx8fEATJgwgXHjxnH06FF8fHx0oz8biE1MZsiSHbqc1cG4uAjTHmxOy2oVeO7LCHafibd3SMWKTROEiHQXkSMickxEXraw3EdEvhaRvSKyQ0SamC0bJyIHRGS/iHwhIp62jNVWTp06RYMGDRg2bBhNmjThkUceYdOmTXTq1Ing4GB27Nhx0/iaNWvSrFkzXFxu/l/z008/0bVrV3x9ffHx8aFr1678+OOPKKX49ddf6d+/PwCPP/4469atuyWOBQsW0KNHD65f16WB+WXenVWXszoeT3dXFgwOoWI5T4YvC+ds3DV7h1Rs2OwahIi4Ah8DXYFIIExE1iulzBuqTAQilFJ9RKSBaXwXEakKPAs0UkpdF5GvgIHAksLEFPrtAQ6eTyjMJm7RqEo5ptzfOMcxx44dY9WqVcyfP582bdrw+eef89dff7F+/Xreeustevfunet+zp07R7Vq1W48DwoK4ty5c8TGxlKhQgXc3Nxuet3c7Nmz2bhxI+vWraNUqVL5f5MlmHk565xHWulyVgflV6YUnz7Rhr5z/mbokjBWP9XxxhwcWsHZ8giiLXBMKXVCKZUCrAQeyDKmEfALgFLqMFBTRCqalrkBXiLiBpQGztswVpuqVasWTZs2xcXFhcaNG9OlSxdEhKZNm3Lq1Kk8bcNSlYaIZPt6puXLl/PDDz+wZs0anRwKYPrP/9woZ71Xl7M6tLqBZfjksRBOxV7l6RU7SdXlr4VmyyqmqsBZs+eRQLssY/YAfYG/RKQtUAMIUkrtFJEPgDPAdWCjUmqjpZ2IyAhgBBjn73OS2zd9WzH/w+zi4nLjuYuLC2lpeetzHxQUxObNm288j4yM5M4778Tf359Lly6RlpaGm5sbkZGRVKlS5ca4Jk2aEBERQWRkJLVq6SZn+fFV+Flm/3ZMl7M6kQ51/Hi7bzNeWLWH//t6P+/0a6orzQrBlkcQlv6vZP26+w7gIyIRwDPAbiBNRHwwjjZqAVUAbxF51NJOlFLzlVIhSqmQgIDiO8l5t27d2LhxI/Hx8cTHx7Nx40a6deuGiHDXXXfdqHhaunQpDzzw34Fay5Yt+eSTT+jVqxfnzzvtQViR23Jcl7M6q/6tg3j2f3X5Mvwsc38/bu9wnJotE0QkUM3seRBZThMppRKUUkOUUi2AwUAAcBK4GziplIpWSqUCa4GONozVYYSFhREUFMSqVasYOXIkjRsbRz2+vr5MmjSJNm3a0KZNGyZPnoyvry8A7777LtOnT6du3brExsby5JNP3rTN2267jQ8++ID77ruPmJiYIn9PzuZYVCKjlhvlrB8/ostZndG4rvV4oEUV3vvxCBv26i9GBWWzdt+mawf/AF2Ac0AY8LBS6oDZmArANaVUiogMB25XSg0WkXbAp0AbjFNMS4BwpdRHOe1Tt/suGP0Z/Sc2MZnec/7meko6Xz/dSVcsObHktHQeXbidPZGX+WJ4O1rX8LV3SA7JLu2+lVJpwBjgJ+AQ8JVS6oCIjBKRUaZhDYEDInIY6AGMNa27HVgN7AL2meKcb6tYNQ1M3VmX79TlrMVEKTdX5j8WQtUKXgxftpPTsVftHZLTsemxs1Lqe6VUPaVUHaXUm6bX5iml5pkeb1VKBSulGiil+iql4s3WnWJ6vYlS6jGlVLItY9VKtsxy1p2n45mhu7MWGz7eHnz6RBsylGLIkjAuXUuxd0hORZ9c1TT+K2ed0F2XsxY3tfy9mf9YCJFx1xm5fCcpabr8Na90gtBKvFWmctYBIdUY1VmXsxZHbWv58v6Dzdh+Mo6X1+7V3V/zSHdz1Uq0LcdjmPj1PjrV9eONPrqctTh7oEVVTsdeY/rP/1DTz5tnuwTbOySHpxOEVmJllrPW9PNmziOtdTlrCfDM/+reSBLVfUvTu2VVe4fk0PRvhAPIa7tvzXpiE5MZuiTsRndW3benZBAR3u7blPa1fXlp9V52nIyzd0gOTScIB5Ndu2/NejLLWS8mJOly1hLIw82FTx4NIcjXixHLwzkRnWjvkByWThA2Zq1235s3b6Zz58489NBD1KtXj5dffpkVK1bQtm1bmjZtyvHjuqVAXmRkKF5cvVeXs5Zw5Uu7s+SJtriKMHRJGHFXdfmrJSXrGsQPL8O/+6y7zUpNocc7OQ6xRrtvgD179nDo0CF8fX2pXbs2w4YNY8eOHcyaNYuPPvqImTNnFv79FHMzNv3Dt3vO63JWjep+pZk/OIRBC7YxYlk4nw1rh6e7q73Dcij6CKIIWKPdN0CbNm2oXLkypUqVok6dOtxzzz0A+d5OSbV6ZyQf/arLWbX/tK7hw/SHmhN+Op6XVuvy16xK1hFELt/0bcUa7b6tuZ2SaOvxWF5Zu1eXs2q36NmsCmfirvHej0eo6Vea8ffUt3dIDqNkJQitRDoencioz3ZSQ5ezatl4qnMdTsdc48Nfj1Hdz5v+rYPsHZJD0L8pDia7dt9awcQmJjNkcRhuLsJiXc6qZUNEeKNPE26r688ra/ey5bhuiw82bPdtD7rdd8EU188oKTWdRxZuZ/+5y3wxoj2tdMWSlouEpFT6zdnCxYQk1j7dibqBZewdks3Zpd23ptlT1nJWnRy0vCjn6c6nT7TBw82VIUt2EJNYsptI6wShFUuZ5awvda+vy1m1fKnmW5qFj4cQfSWZ4cvCSUpNt3dIdqMThFbsmJezPtW5jr3D0ZxQi2oVmDmgJRFnL/H8V3vIyCg+p+LzQycIrVjR5ayatXRvUomJPRry3b4LvL/xiL3DsQtd5qoVG7qcVbO2YbfX4lTsVeZuPk4N39IMbFvd3iEVKZ0gtGIh7moKQ5foclbNukSE0F6NiYy/zqvr9lPVx4vbgwPsHVaR0V+xHEBe230vXbqU4OBggoODWbp0qT1CdUhJqemMWBbOv5eTWPC47s6qWZebqwuzH25JcGAZnv5sF/9cvGLvkIqMTROEiHQXkSMickxEXraw3EdEvhaRvSKyQ0SamC2rICKrReSwiBwSkQ62jNVRZNfuOy4ujtDQULZv386OHTsIDQ0lPj7eTlE6DqUUL63eS/jpeKY/pMtZNdsoayp/9fJwZcjiMKKuJNk7pCJhswQhIq7Ax0APoBEwSEQaZRk2EYhQSjUDBgOzzJbNAn5USjUAmgOHbBWrLVmr3fdPP/1E165d8fX1xcfHh65du/Ljjz/eWGfixIl06NCBkJAQdu3aRbdu3ahTpw7z5s0rsvdqDzN+/of1pnLW+5rpclbNdqpU8OLTJ9oQdzWF4UvDuZ5S/MtfbXkNoi1wTCl1AkBEVgIPAAfNxjQC3gZQSh0WkZoiUhG4DtwBPGFalgIUumH7uzve5XDc4cJu5iYNfBswoe2EHMdYo933uXPnqFat2o3nQUFBnDt37sbzatWqsXXrVsaNG8cTTzzB33//TVJSEo0bN2bUqFEFfn+ObM3OSD7U5axaEWpStTwfDWrJ8OXhPPflbuY80hpXl+JbKWfLU0xVgbNmzyNNr5nbA/QFEJG2QA0gCKgNRAOLRWS3iCwUEW9LOxGRESISLiLh0dHR1n4PVmGNdt+WWqKYl3D26tULMFp/t2vXjrJlyxIQEICnpyeXLl2yxttwKFuPx/KyLmfV7ODuRhWZ3LMRPx24yDs/OOWJjTyz5RGEpd/YrH/l3gFmiUgEsA/YDaQB7kAr4Bml1HYRmQW8DEy6ZYNKzQfmg9GLKaeAcvumbyvWaNMdFBTE5s2bbzyPjIzkzjvvvGUf5tvP7z6chdOVs8Ycg7PbQQSQbP5LLsstjLe4LLvX8/tfa2wnv+/J7L+lfcHFcSfvGdKpFqdjr7Hgz5NU9/PmsfY17B2STdgyQUQC1cyeBwHnzQcopRKAIQBifAU8afopDUQqpbabhq7GSBAlVrdu3Zg4ceKNC9MbN27k7bfftnNURc/pylnjT8GC/0HyZXtH4lwqN4chP4CHxRMHDmFSz0acjbvGlG/2E+TjxV31A+0dktXZMkGEAcEiUgs4BwwEbirNEZEKwDXTNYZhwB+mpJEgImdFpL5S6gjQhZuvXRRbYWFh9OnTh/j4eL799lumTJnCgQMH8PX1ZdKkSbRp0waAyZMn4+vra+doi5Z5OesXI9o7fjlrWgqsHgooGPYrePuBUsbzzFOGNz0vyH8p5Ppm21EZhdiGNd6LaTvX4+G3t+CHl+CBj63//8VKXF2EDwe15KFPtjJmxS5WjepIoyrl7B2WVdm03beI3AvMBFyBT5VSb4rIKACl1DxT6eoyIB0jATyplIo3rdsCWAh4ACeAIZnLsqPbfReMM3xGSime+zKCbyLO8/HDrZyjYmnj/8GWj+DBJdC4j72jcS6/vgF/vA995kPzAfaOJkf/Xk6i98d/IwLrRneiYjlPe4eUL3Zr962U+l4pVU8pVUcp9abptXlKqXmmx1uVUsFKqQZKqb7mCUApFaGUClFKNVNK9c4tOWjF24xNR/km4jwvdnOSctZ/fjKSQ8iTOjkUROeXoXpH2DAOYo7aO5ocVSrvyadPtCHheipDl4RxNbn4XPPTrTY0h7dmZyQf/nKUh0KCePpOJyhnvXwOvh4FFZtAt7fIuH6d5OMn7B2VZQ46YZhHrVq49lsI826DVU/AsE3g7mXvsLLVqEo5Zj/SiieXhDF25W4+eSykWJS/6gShObRtJ4xy1o51/HizT1PHL2dNT4M1wyAtGR5cghI3Tg16kOTD1r3/prjzqFmTWuu/waXPJ/D5g/DTq9Bzur3DytFd9QMJfaAJk9bt5/UNB5nay/mnC9YJQnNYx6MTGbncKGed+6gTlLMC/P4OnNlinDv3D+by6tUkHz5MwPjxlKpb197RWeZgOTf1zBkuvv0Ol1Z+ie/gx6Djs7DlQ6h1u8OfrnusfQ1Ox1xl4V8nqeFXmiGdatk7pELRCUJzSE5Xzgpw/Df44wNo+Sg0H0BGUhLRH83Gq3lz/IYPc/yjHwehlOLKb5uJmTuX8n374NplMpzZCuufNcpffWvbO8QcvXJvQ87EXeP1DQep5lOauxtVtHdIBeYEX8m0kiaznPXC5STmD3aS7qxXLsLa4RBQH3q8B0D8Z5+RdvEigS88r5NDPogIgc+PJz0+nrhPF4OrO/RbZNxEt2qIcfrOgbm6CDMHtqBJ1fI888Vu9p9z3ntgdIJwAObtvqdPn06jRo1o1qwZXbp04fTp0zfGlYR230opJqzJ7M7anNY1nKA7a0Y6rB0GyYlGSauHN+mXLhEzfwFl7ryT0qZ7V7S882ralLI9uhO7ZAlp0dHgUwMemAMXImDTVHuHl6vSHm4sfDwEX28Phi4J4/yl6/YOqUB0gnAwLVu2JDw8nL1799K/f39eeukloOS0+zYvZ+3ZrIq9w8mbP6fDyT/g3vcg0LifJGbBAjKuXCFg3Dg7B+e8AseORaWkEDN3rvFCw57QbhRsmwOHv7NvcHkQWNYof72eks7QJWEkOmH5q04QNpbfdt933XUXpUsbp1Tat29PZGQkUDLafTtdOSvAqb9h81vQ9EFo+RgAqRcuEL/8M8o/8ACe9evZOUDn5VGzJhUe7E/8V6tIyWxq2fU1qNwC1j0Fl87YM7w8qV+pLB8/0oqjUYmMXrGLtPQMe4eULyXqIvW/b71F8iHrlhuWatiAShMn5jimoO2+Fy1aRI8ePYDi3+7bvJz1jd5OUM4KcDUG1jwJPjWh54wbDeqiP5oNQMCzz9gxuOIh4OmnubzuG6JmzSJoxgxwKwUPLoZ5d8DqJ2HI98Y1Cgd2R70A3ujdhFfW7mPK+gO80dt5ug/rI4giUJB235999hnh4eG8+OKLQPFu933CVM5a3bc0cx9pjYebE/yzzMgwboa7FmtcdyhVFoDko0e5vG4dPo88gnsVJzlF5sDcAgLwG/IEV374kev79hsv+taGXh9C5A749XX7BphHg9pWZ1TnOqzYfoZFf520dzh5VqKOIHL7pm8r+W33vWnTJt58801+//33G2OLa7vvm8tZ21K+tGN/G7xh62w49jPc+4FRemkSNWMmLt7e+I0YbsfgihffoUOJ/2IlUdOmUX3xp8YXoyZ9jes+f8+CmrdDcFd7h5mrl7rV50zcVd78/hBBPqXp3qSSvUPKlRN8VStZdu/ezciRI1m/fj2Bgf+1D+7WrRsbN24kPj6e+Ph4Nm7cSLdu3ewYaeElp6Uzcnk4503lrNX9nKCcFeBsGPwSCg17QZthN16+tnMnib/+it+wYbj5OEH1lZNwLVMG/6ee4tq2bVz9e8t/C7q/bbQz+XokJJzPfgMOwsVFmP5QC5oHVeC5L3ez5+wle4eUK50gHMyLL75IYmIiDz74IC1atLhx6si83XebNm2cvt23UoqXVu8l7JQTlbOC0Yp69VAoVwV6fXTjuoNSiqgPpuEWEGDc/atZVYWBA3CvWpWoadNQGaYLve5e0H8xpCYZ7U3SHfdIOZOnuysLHw/Bv0wpnlwaTmT8NXuHlDOlVLH5ad26tcrq4MGDt7ym3cwen9H0jUdUjQkb1Oxfjxb5vgssI0OpLx5WKtRXqbPhNy1K2LRJHazfQMV9+aWdgiv+Lq1frw7Wb6Aurf/25gURXyg1pZxSv7xhn8AK4OjFBNVkyo+q6/TN6vL1FLvGAoSrbP6m6iMIrcit3RXJLGcrZwXYMR8Ob4C7QyGo9Y2XVVoaUTNm4FGrFhX69rVjgMVbufvuo1SDBkTPmoVKSflvQfOB0OIRY/6IE5vtFl9+1A0syyePtuZE9FVGr9hFqoOWv+oEoRWp7SdimbBmLx1qO1E5K8D53cYEQPW6Q4fRNy26/M03pBw7TsD4cYhbiar7KFLi4kLg88+TGhlJ/Jdf3bzw3vfBvx6sGQ6JUfYJMJ861vXn7b5N+fNoDJPW7bdYqWhvJSJBOOIH7yiK8rM5EZ3IyM+MctZ5jzpJOStAUoLRA8g7AHrPvXHdATAa8n34EV7Nm1P27rvtGGTJ4H1bJ0q3a0fMnDmkJyb+t8DD2yg3Tk4wemJlpNstxvx4MKQaz/yvLivDzjLvd8ebM8RJfkMLztPTk9jYWJ0kLFBKERsbi6en7adIzCxndRUnK2dVCr4da9y1228RlL65MCCzIV/A8+Od52jIiYkIgS88/18jP3MVGxmNEk9shr8ce+4Ic+O71qNX8yq8++Nhvtt7wd7h3KTYHw8HBQURGRlJdHS0vUNxSJ6engQFBdl0H+blrF8Mb+885awAu5bCgbXQZTLU6HDTovTLl42GfJ074922rZ0CLHm8mjalbHejkZ/Pw4Nw8/f/b2GrwXDqT/jtLajRCWp0tF+geSQivNe/GecvXWfcVxFUKu/pMFV9Ystv1iLSHZgFuAILlVLvZFnuA3wK1AGSgKFKqf1my12BcOCcUqpnbvsLCQlR4eHhVnwHWmEppRj3ZQTrIs7z0aCW3N/cie4uvngAFvwPqneAR9eCy80H3FEffEDsok+ptW6d7rlUxFJOneL4fT3xGTCASpMn3bww+Qp8codR/jrqL/D2s0+Q+RR3NYU+c/4mMSmNr5/uVGRfpERkp1IqxNIym51iMv1x/xjoATQCBolIoyzDJgIRSqlmwGCMZGJuLHDIVjFqtjdz01HWmbqzOlVySE405kL2LA9959+SHFIvXCBu2XLdkM9OPGrWpMJDDxL/1VekmLXEB4y2Jw8ugWsxxk10GY5ZIZSVr7cHi59oQ1qGYsiSHVy+lmrvkGx6iqktcEwpdQJARFYCDwAHzcY0At4GUEodFpGaIlJRKXVRRIKA+4A3gfE2jFOzka93G+WsD7Z2snJWgO9fhJijMPgbKBN4y+Lo2bNBKQKeGZPrpr48/CW/nf3NFlFahwNeOrm96u080vCRHMdkNvKLnjWLqtOzXHOo3By6vQXfv2C0Ren0rA2jtZ7aAWWY/1hrHl20nVGf7WTp0LZ2LeawZYKoCpw1ex4JtMsyZg/QF/hLRNoCNYAg4CIwE3gJKGvDGDUb2X4ilpdWG+Wsb/ZxonJWgIjPYc/n0HkC1O58y+Lko0e5/PU6fAcPxr1q1Rw3tTtqN29uf5Pq5apT3qO8rSIuMIXjFW9cSbnCOzveoW6FurSrnPVPxn/cAgLwfeJxYufOw3fIULyaNrl5QJthRr+mX0KN04TVnGPipna1/XivfzPGfbmHiV/v4/3+zez2+2PLBGHpHWX91/gOMEtEIoB9wG4gTUR6AlFKqZ0icmeOOxEZAYwAqF69eiFD1qzhZMxV5yxnBYj+B757HmrcZiQIC6JmzMSldGn8Ro7IcVMp6SlM3TKVyt6V+arnV5R2d6KL83aUlJZEv/X9CN0ayppea/By88p2rN+TT3Lpi5VETZ9GjcVZqppEjHYon0QY7VFG/QFejnHxNzd9WgZxOvYaMzcdpYZvaZ7pEmyXOGz5mxsJVDN7HgTc1FFLKZWglBqilGqBcQ0iADgJdAJ6icgpYCXwPxH5zNJOlFLzlVIhSqmQgIAA678LLV/ir6YwZPEO5ytnBUi9blx3cPeCfgvAxfWWIdd27TIa8g0fnmtDvgX7FnDi8gkmdZikk0M+eLp5MqXDFM5eOcvcPXNzHOtapgz+Tz/Fta3bSPz771sHeFWA/kvgygX4ZoxRtuwkxnYJpm/Lqkz7+R++iTiX+wo2YMsEEQYEi0gtEfEABgLrzQeISAXTMoBhwB+mpPGKUipIKVXTtN6vSqlHbRirZgXJaemMcMburJl+fAWiDkCf+UYzvixUPhryHY0/ysJ9C+lZuye3Vb3NVhEXW20rt6VfcD+WHVjGwdiDOY6tMHDgrY38zAW1hq6hRpuUHfNtFLH1iQhv92tKu1q+vLhqL2Gn4oo8BpslCKVUGjAG+AmjEukrpdQBERklIplTnDUEDojIYYxqp7G2ikezLaUUL6/ZR9ipeKY96ETdWTPtXwM7F0On5yDY8h3Rib/9xvVdu/AfMwYXr+xPe6RnpDN1y1TKupflpTYv2Sjg4m9c63H4ePowdctU0jKy79Tq4uFBwNhnST54iITvf7A8qP3TRpuUjf9ntE1xEqXcXPnksdYE+XgxYlk4J2OuFun+bXpyWCn1vVKqnlKqjlLqTdNr85RS80yPtyqlgpVSDZRSfZVS8Ra2sTkv90Bo9jXrl6N8vfuc85WzAsSdgPVjIagt/O//LA5RaWlETZ9ulFf2y7kh38ojK9kbs5cJbSfg4+lkidKBlC9VnlfbvcqhuEMsO7gsx7Hleva03Mgvk4jRJsU70DiNmHTZNkHbQIXSHiwe0gYRYeiSMOKvWnh/NuJEVw81R/X17khmbjpKf2csZ01LNv5guLhC/0XZzm+c14Z85xPPM2vXLG6rehv31rrXRkGXHHfXuJsu1bswJ2IOZxLOZDvOaOQ3ntSzZ4n/apXlQaV9jf/Hl84a7VOc6HpEDT9v5j/WmnOXrjNy+U6S04qm11SOCUJEyuWwTJcMaew4GceE1fvoUNuPt5ytnBXg58lwYQ/0ngMVLP+TzkhKIvqj2Xg2b0bZrtlPbamU4rWtrwEwuf1k5/ssHNTEdhPxcPEgdGtojj3VvG+7zayRXzanYqq3N44SD3xtnFJ0IiE1ffngwebsOBXHhNV7i6S/XG5HEJszH4jIL1mWrbN2MJpzORlzlRHLwwny9XK+claAQxtg+zxo9xQ0uC/bYfErVpD2778EPv98jn/0vzv5HX+f/5uxrcZSuUxlW0RcIgWWDmR8yHh2/LuDr499ne24G4384uKIy1ryaq7Tc1Dnf/DDy/Dv/uzHOaBezavwYrf6rIs4z8xNR22+v9x+o81/G7LOb6m/HpVgmeWsLiIscbZyVjC6s37zNFRuYVS4ZCP98mViPpmPd+c7cmzIF5cUx7s73qVZQDMG1h9og4BLtr7BfQmpGMIH4R8QfS37xpteTZtStls3YhcvJi0mxvIgFxejUs3Lxzi9mJxoeZyDevrOOjwUEsSsX46yZmekTfeVW4JQ2Ty29FwrIYzurDs5fzmJBYNbO185a3qqceOUUvDgYnArle3Q2AULyLhyhcDxOXd7eS/sPRJTEwntEIqrhfsntMJxERemdJhCcloyb+94O8exAc+NRSUnEzMnh3soygRAv4UQd9xox+FERIQ3+zSlU10/Xl67l63HY222r9zupA4UkfEYRwuZjzE9LzZ3pb24ag8pZlP+WTq1ZykbWjoHeMsrFrdlYT1L4/Kwbt5jtfBiAbd1MSGJA+cT+GhQS1rXyHpg6QR+fR0iw4wJ731rZzss9d9/iVv+GeV79cKzfv1sx/0Z+SffnfiOp5o/RV2furaIWANqlq/JUy2eYtauWfxy+he61OhicVypWrWo8GB/4r/6Ct8nHscjuw4LtW437pbf/DbUvB1a5tz7yZG4u7ow55HW9Ju7hZHLw1n7dCfqBpax+n5ybPctIlNyWlkplf2xuR0UtN13txl/3FIVYOlcs8VzahZetDQu6/Ysj7G0LQtx5OHkXl7jt7jPPMTxUJtqPNa+Ru6BOJqjP8OK/hAyFHrOyHHo+f/7PxK+WU+dH3/ItufS1dSr9PmmD15uXqy6fxUerh4Wx2nWkZqRyqANg4hLimNd73WU87BcR5MaFcXxbt0pe9ddVJ0+LfsNZqTDsgfg3E4Y/hsENrBR5LZxNu4afeb8TWkPN34YezvepfLfPSmndt8opYrNT+vWrZWmZevyOaXeraXUnI5KpVzLcWjS0aPqYMNG6t+338lx3Nvb31ZNlzRVuy/utmKgWk72R+9XzZY2U1O3TM1x3MWZM9XB+g3UtX37c95gwgWl3q2t1MftlUq+asVIi8au03Fq8V8nCrw+EK6y+ZuaW5nrcBEJNj0WEflURC6LyF4RaZnvVKVp9pKeBmuGGZPI9F9s9FvKQV4a8u2J3sPnhz5nYIOBtAhsYeWAtew09m/M4EaDWf3PasL+Dct2nN+TT+JaoQLROR1BAJStZMz5EXUQfrTcoNGRtazuwxOdatlk27ldpB4LnDI9HgQ0B2pjzM/woU0i0jRb+P1dOP039JwOATlP8HNt1y4Sf/kFv2HDsm3Il5qeytQtU6noXZGxrXSHmKL2dIunCSoTROjWUJLTky2OcS1TBv+nRnF1y1bLjfzM1e0Ct42HXctgbzY32pVAuSWINKVU5rRGPYFlSqlYpdQmwNu2oWmalZzYDH+8Dy0egeY5l6CqPDbkW7h/IccuHWNS+0l4u+tfhaLm5ebF5A6TOZ1wmnl75mU7rsKgQbhXqUL0tOmWG/mZu+tVqNYeNjwHscetG7CTyi1BZIhIZRHxBLoAm8yW5XyMrmmOIDEK1gwH/3pw7/u5D/9t838N+UpbLt89fuk48/fOp0etHtwRdIe1I9byqEOVDvSu25vF+xdzJO6IxTEuHh4EPDeWpIMHSfghm0Z+mVzd/mu3supx43RkCZdbgpgMhGOcZlqvlDoAICKdgRO2DU3TCikjA9YOh+QEY45ij5y/6av0dKKmT8uxIV+GymDqlql4u3szoY3zna8ubl4IeYHypcozZcuUbDu+3mjkNzObRn7mygcZTf3+3Wd0fi3hckwQSqkNGNOANlRKDTdbFA4MsGVgmlZof003Ti/1eA8qNsp1+OV1poZ847JvyPflkS+JiI5gQpsJ+Hn5WTlgLb/KlyrPK+1e4UDsAVYcWmFxjLi4EDh+XM6N/MzV7wEdxkDYAjj4jZUjdi653QeRY19jpdRaq0dUCAW9D0Irhk5vgSX3QeO+xh2zudw8kpGUxPHuPXALDKTmlyst3kdyIfECvb/pTYvAFsy7e55uxucglFI8+9uzbDu/jbUPrKVa2WoWx5x5/AmSjx2jzsaNuJbJ5bpRWgos7g4xx4ypSn1q2iZ4B5DTfRC5nWJaDfwfxgXqnsD9Zj96jgbNMV2NhdVPGr/UPWfk6c7CGw35XrDckE8pxevbXkehmNxBd2p1JCLCq+1exdXFlde2vmaxw8FNjfyWLMl9o24e0P9T4/GqIUbCKIFySxD9gH+AZhhzRb+pjDmkhyilhto8Ok3LL6Vg3VNwLca438Ez2471N6RfvkzM/AU5NuT78dSP/HnuT55p+QxVy1i+q1qzn0relRjXahzbLmzjm+OWTwt5NWtG2W7diPv00+wb+ZnzqQkPzIbzu+AXh2oaUWRyuwbxtVJqINAZOA5ME5G/TBepNc3xbJ0NR3+Ce96EKi3ytErswoVkJCRk25DvUtIl3tnxDk38mvBwg4etGKxmTQ/Wf5BWga14P+x9Yq5bTgABz40lIzmZmLnZl8bepFEvaDvC+Hd1JJcqqGIorw38k4DLQALG/Q+eNotI0woqMhw2TYUGPaHt8FyHg6kh37LllO91f7YN+d4Pf5+E5ASmdpyqO7U6MBdxYUrHKVxPu867O961OKZUrVpU6N+f+C+/JOVM9jPU3aTr61CpmXFketm27bUdTW6tNu4SkfnATuAuYJZSqqVS6qciiU7T8up6vHGuuFwV47RAHq8RRM+eDRkZ+D/zrMXlW85tYf3x9QxtOpT6vtl3dNUcQ+3ytRnZbCQ/nvqRzWc3WxzjP/ppxN2d6Fl5bAbh7mmUSWe2iU9PzXWV4iK3I4hfgLbAX0ApYLCIfJj5k9vGRaS7iBwRkWMi8rKF5T4i8rWpt9MOEWlier2aiPwmIodE5ICI6F4GWvaUgvXPwJXzxnUHL8vtMbJKPn6cy2u/xufhh/EIuvW6wrXUa7y27TVqlqvJiGbZ92TSHMvQJkMJ9gnm9W2vk5hy62RA7oGB+D4+mITvvuP6gQN526hfHbh/FpzdDr+9ZeWIHVduCWIIMAMIw7j3YWeWn2yJiCvwMdADaAQMEpGsxegTgQilVDNgMDDL9Hoa8LxSqiHQHhhtYV1NM4QthEPfwt1TIchy12JLombMMBryjRppcfnsiNmcSzxHaMdQSrlmP6mQ5ljcXd0J7RBK9LVoZu6aaXHMjUZ+06bnfcNN+0Orx437a45tyn18MZDbReql2f1gNl91NtoCx5RSJ5RSKcBK4IEsYxphHKWglDoM1BSRikqpC0qpXabXrwCHAF06ot3qwh74aSIEd4P2o/O82rVdu0nclH1Dvn3R+1hxaAUD6g+gVcVW1oxYKwJNA5rySMNH+PLIl+y6uOuW5a5ly+I3aiRXt2zJvZGfue7vQGAjWDsSEi5YMWLHlOtFahHpICL9RSTQ9LyZiHyOcdopJ1WBs2bPI7n1j/weoK9pu20x7toOyrL/mkBLYHs28Y0QkXARCY+Ozn6uWq0YSr5izClc2t9oj+CSt5oLpRRR06bhGuBvsSFfanoqU7ZOwd/Ln+daPWfdmLUik1mSPHXrVIsdX30efjjvjfwyeZQ2rkekXjPauGSk57qKM8vtIvX7wKcY90N8Z5ph7meMP9bBuWzb0lXCrHewvAP4iEgE8AywG+P0Uub+ywBrgOeUUgmWdqKUmq+UClFKhQQEFJtZULXcKAXfPgfxp4wGa955b3uR+Ntmru/cScBoyw35Fh9YzNH4o/xfu/+jjIf1p3HUikZp99JMaj+Jk5dPsmDvgluWu3h4EDD2WZIOHuTKjz/mfcMB9eG+aXDqT6NLcDGW21eu+4CWSqlBwD3Ay8BtSqlZSqncWh1GAub3vAcB580HKKUSTDfdtcC4BhGAcUMeIuKOkRxWOFpLD80B7F4O+1fDXROhRsc8r6bS04meMT3bhnwnLp9g3p55dKvZjbuq32XNiDU76FS1E/fXvp9F+xbxT/w/tywv17MnperXJyovjfzMtXgYmg+Cze/AyT+sGLFjyS1BXM9MBEqpeOCIUupoHrcdBgSLSC0R8QAGAuvNB4hIBdMygGHAH0qpBDH6GCwCDiml8nEVSSsRLh6E71+C2ncak7zkw+Vv1pN89JjRkM/d/aZlGSqD0C2heLl58XLbW4ruNCf1UpuXKFeqHFO3TCU9yykhcXUl8PnxpJ45Q/yqfE4UdO8H4B9szFSYWDxPb+eWIOqIyPrMH4yLyOtF5FvT82wppdKAMcBPGBeZv1JKHRCRUSIyyjSsIXBARA5jVDtllrN2Ah4D/iciEaafewv4HrXiJOWqcd2hVFnouwDyceNaRlIS0R9+iGezZpS9p+sty1f/s5pdUbt4sc2L+Hv5WzFozZ4qeFZgQpsJ7IvZxxeHv7hlufftt1O6TRti5swlPfFq3jdcqoxRVp10Gb4eYbSXL2Ys9zT+T9aqo2n8dx0h1zuRlFLfA99neW2e2eOtWLiWoZT6Ky/b10qg71+CmH9g8DooE5ivVeNXfE7av/9S5d13b2m2d/HqRabvnE67yu14oE7Wf/aas+tRqwcbTmzgw90fclf1u27qp5XZyO/UgIHELVlCwJi8V8NRqYlR2bThOfh7Btz+vPWDt6PcjiAqAE2UUr8rpX4H3geWAkuA/P12alph7VkJEZ/BHS8ap5fyIT0hgZj58/G+43a8293ckE8pxRvb3yA9I50pHaboTq3FkIgwqf0kBOH1ra/f0vHVq3lzyt5zj9HILzY2fxtv/YTRVv7XN+H0VusF7QBySxAvcfN1Aw8gBLgTGGVpBU2ziZijsGE81OgEnfM/k1vsguwb8m08vZHNZzczpuUYi3MJaMVD5TKVGdtqLH+f/5sNJzbcsjzguefy18gvk4hxl3WF6rDmSbgWZ6WI7S+3BOGhlDK/l+EvpVSsUuoMRtM+TbO91OvGdQe3UsbkP665nRnNsvrFi8QtW2Y05GvQ4KZll5Mv89b2t2jk14hHGj5ixaA1RzSwwUCaBzTnvbD3iEu6+Q95qdoFaOSXybOccX/E1WijqV8OE7E5k9wSxE23mCqlxpg91TcdaEXjp1fh4n7o84nRjC+fYnJoyPdB+AdcTr5MaMdQ3Fzyl3g05+MiLoR2DCUxNdFix1f/p59GXF3z3sjPXJUWcM8b8M+PsPXjwgfrAHJLENtF5Ja+ySIyEthhm5A0zcyBryF8EXR8Furdk+/Vk48f59Katfg8POiWhnxbz29l3bF1DGkyhAa+DbLZglbc1KlQhxFNR/D9ye/5I/LmexjcKwbi+/jj+WvkZ67tCKPd/KYpEJljuzqnkNuc1IHAOiAZyGxo0hqjs2tvpdRFWweYH3pO6mIm7gR80tm4c3XID+Dqnvs6WZwdM4ZrW7dRZ9PPN/Vcup52nb7f9MXVxZXV96/G001PcVKSpKan8tCGh0hMTWTdA+vwdv/vjHn6lSscv7srnk2aUH3Rwvxv/Ho8zLvDqMMc+Sd4VbBa3LZQ4DmplVJRSqmOwOvAKdPPa0qpDo6WHLRiJi3ZmN9BxJgbuADJ4dpuU0O+4bc25JsTMYfIxEimdJiik0MJ5O7qzpQOU7h49SIf7rr5dJJr2bL4PTWKq3//zdUtW/K/cS8f499swnlYP8apr0fkqbuZUupXpdRHpp9fbR2UprFpKlyIgAfmGNUh+XRzQ77BNy07EHuAZQeX0b9ef9pUamOdeDWn0yKwBYMaDOKLw18QERVx07LMRn5R+WnkZ65aG+gyxWhDH1aAoxAHkdcpRzWt6Bz+HrbNgXajoGHPAm0icfNmrofvJGD06Jsa8qVmpDLl7yn4efoxrvU4a0WsOalnWz1LJe9KTNkyhZT0/3oxuXh44P/sMyQdOJC/Rn7mOowx2tD/NNFoS++EdILQHMuls0aZYOXm0PW1Am1CpacTPX06HjVqUKFfv5uWLT2wlCPxR3i13auU8yhnjYg1J+bt7s2k9pM4cfkEi/YtumlZ+fvvp1S9evlv5JfJxcVoQ1/a3yjTTrLYkNqh6QShOY70VONGo4x0o8eNW8FmccuuId/phNPMjZhL1xpd6VKji7Wi1pzc7UG3c2+te5m/bz7H4o/deP2mRn6rVxds495+Rjv6+NNGOw4nux6hE4TmOH5705jz9/6ZxhzABZCRnEz0Rx8ZDfm6/VcWm6EymLplKqVcS/FK21esFLBWXExoO4Ey7mWYsnXKTR1fve+4w2jk9/EcMq7mo5GfuRodjbb0+9fArmVWirho6AShOYZjm+CvGUZfm6b9C7yZ+BWfk3bhAoHPP39TT6W1R9cSfjGc50OeJ6C0vsdTu5mvpy8vtXmJvdF7+fLIlzdez2zklx4bS+ySJQXfwW3jofZd8MNLRrt6J6EThGZ/CReMOX4DGxudMQsoPSGBmE8+wfv2mxvyRV2LYnr4dNpWakvf4FsnCdI0gJ61e9KpSidm7ZrFhcT/5pv2at6csl27EreoAI38Mrm4QN/5UKqccT0ipYBHI0VMJwjNvjLSjbl9U68ZvWzcvQq8qRsN+Z6/uSHfW9vfIiUjRXdq1XIkIkzuMBmF4vVtN3d8DRg3zmjkN++Tgu+gTCD0W2C0q//+RStEbHs6QWj29ft7xty+902HgHoF3kxmQ75y9/e8qSHfptOb+OXMLzzd4mmql8v//RRayVKlTBWebfksf577kx9O/nDj9VK1a1GhXz/iV64k5ezZHLaQi9p3QueXIGIFRNw6eZGj0QlCs58Tv8Pv70Lzh6HFoEJtKmb2x5CRQcCzY2+8lpCSwJvb36Shb0MGNxqcw9qa9p9BDQbRzL8Z7+x4h/ik+Buv+48eXfBGfuY6T4Aat8F3z0P0rfNkOxKdIDT7SIw2Ti35B8O97xdqU8knTnBpzZpbGvJND59OfFI8UztO1Z1atTxzdXFlasepXEm5wvth//3bdK8YiO/gwSRs2EDSwUJcaHZxNU41uXsa1yNSrxc+aBvRCUIrehkZxhy+SZeN+x1KlSnU5qJnzMDFywu/Uf/NYbXjwg7WHF3D4MaDaeTXqLARayVMsE8wTzZ9km9PfMvf5/6+8brfsCdxLV+eqGnTC7eDclWgz3yIOgA/Om7ZtU4QWtH7eyYc/9WoWKrUpFCburZ7N1d+3oTfsCdvNORLSksidGso1cpW46nmT1khYK0kGtFsBLXK1+K1ra9xLfUaAK7lyuE3ytTIb2shpxcNvhs6PQc7Fxv3SDggmyYIEekuIkdE5JiIvGxhuY+IfC0ie0Vkh4g0yeu6mpM6sw1+fQOa9DPueSiEGw35/P3xffzxG6/P3TOXM1fOMKXDFLzcCl4VpZVsHq4ehHYM5fzV83y0+6Mbr/s8PAi3KpWJ+mBawRr5mfvf/0G1drB+LMQeL2TE1mezBCEirsDHQA+gETBIRLIe608EIpRSzYDBwKx8rKs5m2txsPpJoztrz5lGK+9CSPz9d6Mh35j/GvIdij3E0gNL6Rvcl3aV21khaK0kaxnYkgH1B7Di0Ar2Ru8FwKVUKQKefdZo5PfTT4Xbgas79FtkXJdYPcRoc+9AbHkE0RY4ppQ6oZRKAVYCD2QZ0wj4BUApdRioKSIV87iu5kyUgnVPw9Uo434Hz8I1ylPp6URPu7khX1pGGlO2TKFCqQqMbz0+ly1oWt481+o5AksHMmXLFFLTUwFTI7/gYKJmzESlphZuBxWqGU39LuyBnydbIWLrsWWCqAqYFwxHml4ztwfoCyAibYEaQFAe18W03ggRCReR8OjoaCuFrlndtjnwzw/GnL1VWhR6c5fXf0vy0aMEjHvuRkO+5QeXcyjuEBPbTaR8qfKF3oemAZTxKMOk9pM4dukYn+7/FDAa+QVkNvJbtarwO2lwL7R/GrbPM+aQcBC2TBCWzh9kbWX4DuAjIhHAM8BuIC2P6xovKjVfKRWilAoJCNA9dhxS5E74eYoxV2/bEYXeXEZyMtEffohn06aU7dYNgDMJZ/g44mP+V+1/dK3RtdD70DRznat1pnvN7nyy9xNOXD4BQJnOnSkdEkLMnLkFb+Rn7u5QqNISvhltdH91ALZMEJFANbPnQcB58wFKqQSl1BClVAuMaxABwMm8rKs5ieuXjHOrZSvDA7MLfd0Bbm3Ip5Tita2v4e7izsR2E3U7Dc0mJrSdgJebF6FbQslQGf818ouJIXbp0sLvwM3DKPtWClYPNdrf25ktE0QYECwitUTEAxgIrDcfICIVTMsAhgF/KKUS8rKu5gSUgm+fhYRzxhy9Xj65r5OL9IQEYjMb8rU3LkKvO7aO7f9uZ1zrcVT0rljofWiaJf5e/rzU5iV2Re1i1RHjtJJXixZGI7+Fi0iLiyv8TnxrQa8P4Vw4/BJa+O0Vks0ShFIqDRgD/AQcAr5SSh0QkVEiknlHU0PggIgcxqhYGpvTuraKVbOR8EVw8BvoMtmYo9cKYhcuIt2sIV/M9RjeD3+f1hVb079ewduEa1pe9KrTi/aV2zNj1wz+vfovAAHjniMjKYmYufOss5PGfSDkSdjyEfxTyCqpQhLlZDMc5SQkJESFh4fbOwwN4MJeWHg31O4Mg7402h0XUurFixzv1p2yXbtS9f33ABi/eTy/n/2d1b1WU6t8rULvQ9Nyc/bKWfqt70e7Su348H8fIiJcmDSZS+vWUef77/CoVi33jeQmNcn4/Uk4B6P+gvIWa3SsQkR2KqVCLC3Td1Jr1pd8xbjuUNoXes+zSnIAoyGfSk8nYOyzAPxy5hd+Pv0zo5qP0slBKzLVylZjdIvRbI7czE+njW/4/mNMjfw+/CiXtfPI3dMoB09LNqbhTU+zznbzSScIzbqUgg3jIe6EcQOQt59VNnujId+ggXgEBXEl5QpvbXuLej71eKLJE1bZh6bl1SMNH6GxX2Pe3v42l5Mv416xotHI79tvC9fIz5x/XWP63TNbYfPb1tlmPukEoVnX7s9g31dw50So2clqm42eMRMXLy/8TQ35ZuycQUxSDKEdQ3F3cbfafjQtL9xc3AjtGEpCcsKNjq83GvlNn2G9HTV7CFo+Bn9OM/qXFTGdIDTriTpkzJRVqzPcbr07ma9HRHDl55+Nhny+voT/G86qf1bxaMNHaeJfuGZ/mlZQ9X3rM6TJEL45/g1bz281GvmNHMnVv/7i6rZt1ttRj/cgoD6sHQFXLlpvu3mgE4RmHSnXjN72pcpA3wVGbxkrUEpx8YMPbjTkS05PJnRrKFXLVGV0i9FW2YemFdTI5iOpWa4moVtDuZ52HZ9HHsatsqmRn7UKgDxKG9cjkhNh7TBjmt4iohOEZh0/vATRR4zkUNZ69yLcaMg3+mlcSpfmkz2fcCrhFJM7TKa0e2mr7UfTCqKUaymmdJjCucRzfLz74/8a+e3fX/hGfuYCGxoTa538wzjdVER0gtAKb+9XsHs53PEC1LnLapvNbMjnXqM6Ffr350jcERbvX0yvOr3oWKWj1fajaYURUimEB+s9yPJDyzkQc4DyvTIb+c0ofCM/cy0fhWYDjAvWp/6y3nZzoBOEVjgxx2DDOKjeETpbd9qOy98aDfkCx40jw9WFKVumUK5UOV4MedGq+9G0whrXehz+nv5M2TKFNMkwGvmdPsOl1auttxMRuG8a+NaGNcPgaoz1tp0NnSC0gktNgtVPgKsH9FsIrtab9zlrQ77PDn3GgdgDvNL2FSp4VrDafjTNGsp6lOXV9q9yJP4ISw8spUznzniFtCb64znWaeSXqVRZ43rEtTj4eqQxfa8N6QShFdzG/4N/90GfeVa/0zP+8y9IO3+BwOfHE5kYyezds7kz6E661exm1f1omrX8r7rRSXhuxFxOJ5wm8HmjkV/csmXW3VGlptD9bTi2CbZ8aN1tZ6EThFYwB9ZB2ALo+AzUs+4f7fSEBGLnzcP7ttso3a4dr299HVcXV15t/6ru1Ko5tIntJlLKrRShW0PxbNGcsl3vJtZajfzMhQyFRr3hl9fgzHbrbtuMThBa/sWdhPXPQNUQ+J/1Z8CKXbiI9MuXCXx+POuPr2frha081+o5KnlXsvq+NM2a/L38eTHkRcIvhrPm6BoCxo0j4/p1YuZZqZFfJhGj62uFakYrjmtWTkAm1jtp7My+fMw0F6wyWkWA2WPT88zHtywnl+W5rZ/TOln3k49tZrucbJbnY5up10296z81/mtFqRejiFu2jHL3309izQDeWzeMloEteaj+Q1bdj6bZSu+6vfnuxHdMD59O597fUKFfP+K/WInv4MF4BAVZb0ee5Y35IxbdY0wyNPBzq8y3Yk4nCIDEi5CWBIjpAzZ9yJmPb3zoOS03/WQ2psvTOlmW57hONtss9H4oWGxNHwKfGjl+rAUR8/F/Dfle3fEu19OuM7XDVFxEH+xqzkFEmNxhMn3X9+XNbW/y/uhXuLx+PdGzPrzRhdhqqraCe16HY79A6jXw8Lbq5nWCAHhyo70j0DBryPfww2xRx/jx1I+MbjGa2hVq2zs0TcuX6uWqM7rFaKbvnM7vdXrSfPBgYhcuxG/oEDwbNrTuztqNgrYjrdY12Zz+WqY5jOgZM3Hx9MTrycd4Y9sb1K1QlyebPGnvsDStQB5r9BgNfRvy1va3cB/8EC7lylm3kV8mEZskB9AJQnMQmQ35fJ8cykenlhJ1Lcro1OqqO7Vqzimz42t8Ujwzjy7Af+RIrv75p3Ub+dmYThCa3SmliPpgGq7+/kTe25Ivj3zJIw0foVlAM3uHpmmF0tCvIY83fpy1R9dyrEtd6zfyszGdIDS7u/rHH1wLD8fnqZFMiXibKt5VeKblM/YOS9Os4qnmT1GtbDVCd75NhdGjrN/Iz4ZsmiBEpLuIHBGRYyJyS6MeESkvIt+KyB4ROSAiQ8yWjTO9tl9EvhART1vGqtmHSk8nytSQ76t6sZy8fJJJHSbpTq1aseHp5snUDlM5e+Usy2ucoVRwXaJnzLRuIz8bsVmCEBFX4GOgB9AIGCQijbIMGw0cVEo1B+4EpomIh4hUBZ4FQpRSTQBXYKCtYtXs5/K335L8zz+kDRvAwkNL6Fm7J7dVvc3eYWmaVbWt3JZ+wf1YdugzkoY/SMrp01xas8beYeXKlkcQbYFjSqkTSqkUYCXwQJYxCigrRv+EMkAckDk7txvgJSJuQGngvA1j1ewgsyFfqSaNed1zE2U9yvJSm5fsHZam2cS41uPw8fRhKt/i2boV0bM/tm4jPxuwZYKoCpw1ex5pes3cbKAhxh//fcBYpVSGUuoc8AFwBrgAXFZK6ZsVipn4L4yGfPsebMHeuH1MaDsBH08fe4elaTZRvlR5JrabyKH4w4T1bWCbRn5WZssEYeme76yX7rsBEUAVoAUwW0TKiYgPxtFGLdMybxF51OJOREaISLiIhEdHR1srds3G0q9cIXbuPFw7hPBm2npuq3ob99a6195haZpNda3RlS7Vu/De9W9w7dzRaOQXH2/vsLJlywQRCVQzex7EraeJhgBrleEYcBJoANwNnFRKRSulUoG1gMUpxJRS85VSIUqpkICAAKu/Cc02MhvyLb/d6Gc/uf1k3alVKxEmtpuIu4s7CzpeI+P6dWKt3cjPimyZIMKAYBGpJSIeGBeZ12cZcwboAiAiFYH6wAnT6+1FpLTp+kQX4JANY9WKUOrFKOKWLuXKnS1Y57qXsa3GUrlMZXuHpWlFIrB0IONDxvMj+4m/uxVxn39BSmSkvcOyyGYJQimVBowBfsL44/6VUuqAiIwSkVGmYa8DHUVkH/ALMEEpFaOU2g6sBnZhXJtwAebbKlataMXMmYNKS+PtZqdpFtCMgfV1gZpWsvQL7kdIxRDeaHwUXIToD2078U9BibPc0ZcXISEhKjw83N5haDlIPnGSE/ffz8E7a/BG+/Os6rmKuj517R2WphW5U5dP0W99P14Ir0SLjSep9fVaPBs0KPI4RGSnUirE0jJ9J7VWpKJnziSjlDvTmpxmeNPhOjloJVbN8jV5qsVTzGp4mowyXkRNn27vkG6hE4RWZK5HRHBl40Z+bO9BYOW6DGs6zN4haZpdPd74cYKqNGBdB1eu/vEnV7fZbvrQgtAJQisSSimipk0nqZwnK1teY2rHqXi4Wnc2Ok1zNu4u7oR2DOXrFklc8y1N1DTHauSnE4RWJK7++SfXwsJY0T6FPs0G0SKwhb1D0jSH0Ni/MQObDmZxhySS9u3jyk+Oc0+wThCazan0dC5+8AExfu7s61SZsa3G2jskTXMoo1uO5kT7avxb0Z2oGdMdppGfThCazSVs2EDKP0dZfns6r3aajLe7defN1TRn5+XmxaROU1h6ezqpp89wac1ae4cE6ASh2VhGcjLnZ0zjRCWhQvd7uSPoDnuHpGkOqUOVDgR1683hIOHfj2aRce2avUPSCUKzrbjPP4d/o1l3T1lebv+KvcPRNIf2YtuXWN+tAsTGE7Nkib3D0QlCs530K1e4MOcjImoJvQb8H76evvYOSdMcWvlS5Rnw4GR21BOiFnxi90Z+OkFoNnN67izcrlznwIMt6Vm7p73D0TSn0K1GN44PaI8kpXDqw/ftGotOEJpNpF6MInH5F2xt7MZTD72nO7VqWh6JCE/3fou/Wrhz/at1JNuxkZ9OEJpN7HrnZSQ9A+/Rw6laJus8UZqm5aSSdyUCRo8hQxQRb9pvlkWdIDSriz4cgfePW9nVMYD+d462dzia5pR6dxrGzjsqU+a33fwbsc0uMegEoVndztdfIMUd2r/yAa4urvYOR9Ockou40PmVGVzzhL1v2OcoQicIzaq2bVpOjZ3nON+rDfXrtLV3OJrm1OpUa05Uv9uotj+avzcU/ZQ4OkFoVnM15SrRH0zjShlXurw0y97haFqxcPf46Vwq70bCzNlcSb5SpPvWCUKzmlXLXqHuqWRKDXsUr3I+9g5H04qFUt5lKfvUcGpGprJq/vNFum+dIDSr2HdxD5WWbSIxsAzNho63dziaVqw0fmw0CVUrUOOLP9l9PqzI9qsThFZoqemprJ07nhpRihovvIJ46HkeNM2axNWVOhMmUyUOvv/oBZLTk4tkvzpBaIW2JGIBd/5wntTg6gT07G3vcDStWPLr2p3UJnW58+coFu2YUyT7tGmCEJHuInJERI6JyMsWlpcXkW9FZI+IHBCRIWbLKojIahE5LCKHRKSDLWPVCubE5ROcXDKPwMtQ55UpiIv+zqFptiAi1J0Yim8iRC35lH/i/7H5Pm322ywirsDHQA+gETBIRBplGTYaOKiUag7cCUwTkczzE7OAH5VSDYDmwCFbxaoVTIbK4J1fJtH7rzTc27fBu2NHe4ekacVa6VatKHXnHfTals57GyeRnpFu0/3Z8uteW+CYUuqEUioFWAk8kGWMAsqK0ainDBAHpIlIOeAOYBGAUipFKXXJhrFqBbDqyCqqb9hN2euKqi9OsHc4mlYiVH3hRTxTof6GfXxx+Aub7suWCaIqcNbseaTpNXOzgYbAeWAfMFYplQHUBqKBxSKyW0QWiojFachEZISIhItIeHR0tNXfhGbZv1f/5dPfp9ErXCh33314NW5s75A0rUQoVbcuFXr3ocduWPHbTM4lnrPZvmyZICy171RZnncDIoAqQAtgtunowQ1oBcxVSrUErgK3XMMAUErNV0qFKKVCAgICrBS6lhOlFG9ue5P7f7+Oe4YLAc/pOaY1rSgFPDMGV1d3+v6ewutbX0eprH9arcPNJls1RALVzJ4HYRwpmBsCvKOMd3dMRE4CDYAzQKRSartp3GqySRDWcDjuMBkqw1abL3b2Ru/lyJ7feCoiA59BD+NRrVruK2maZjXulSvj99hjdFq0iHURf7Gh9gbur3O/1fdjywQRBgSLSC3gHDAQeDjLmDNAF+BPEakI1AdOKKViROSsiNRXSh0xjTloq0AH/zCY62nXbbX5YmnKtjK4eKbi//RT9g5F00okv+HDif9qFaO2wPvV3qdL9S6Udi9t1X3YLEEopdJEZAzwE+AKfKqUOiAio0zL5wGvA0tEZB/GKakJSqkY0yaeAVaYqppOYBxt2MT7d7yvjyDywf3wKfz2voffmDG4+fnZOxxNK5Fcy5fHf+QIMt7/gJnlJ1o9OQCIrc5d2UNISIgKDw/P93on+/YjI7lo7kwsDtJjY8HVlTo//YRrGYu1A5qmFYGMpCSOd++BW2AgNb9cWaCZG0Vkp1IqxNIyW55ichoedeugUlLtHYbzCA7GZ+AAnRw0zc5cPD0JeG4sSXv3oZKTEU9Pq25fH0FomqaVYDkdQei+CJqmaZpFOkFomqZpFukEoWmaplmkE4SmaZpmkU4QmqZpmkU6QWiapmkW6QShaZqmWaQThKZpmmZRsbpRTkSigdMFXN0fiMl1lJZJf175oz+v/NGfV/4U5vOqoZSyOFdCsUoQhSEi4dndTajdSn9e+aM/r/zRn1f+2Orz0qeYNE3TNIt0gtA0TdMs0gniP/PtHYCT0Z9X/ujPK3/055U/Nvm89DUITdM0zSJ9BKFpmqZZpBOEpmmaZlGJShAi0l1EjojIMRF52cJyHxH5WkT2isgOEWlijzjtKQ+fUQMR2SoiySLyQn7WLY4K+Xl9KiJRIrK/6CK2v4J+ZiJSTUR+E5FDInJARMYWbeTOQ0RCROTDQm+npFyDEBFX4B+gKxAJhAGDlFIHzca8DyQqpUJFpAHwsVKqi10CtoM8fkaBQA2gNxCvlPogr+sWN4X5vEzL7gASgWVKqRLxZaSQ/8YqA5WVUrtEpCywE+hdnP+N2VtJOoJoCxxTSp1QSqUAK4EHsoxpBPwCoJQ6DNQUkYpFG6Zd5foZKaWilFJhQNZJvPPy+RY3hfm8UEr9AcQVSaSOo8CfmVLqglJql+nxFeAQULVowi56IlJTRA6LyEIR2S8iK0TkbhH5W0SOikhb088WEdlt+m9907p3isgG0+OppqPVzSJyQkSezWsMJSlBVAXOmj2P5NZ/XHuAvgAi0hbjW0xQkUTnGPLyGdliXWdVEt9zYVnlMxORmkBLYLt1wnJYdYFZQDOgAfAwcBvwAjAROAzcoZRqCUwG3spmOw2AbhgJeoqIuOdl526FCt25iIXXsp5feweYJSIRwD5gN5Bm47gcSV4+I1us66xK4nsurEJ/ZiJSBlgDPKeUSrBKVI7rpFJqH4CIHAB+UUopEdkH1ATKA0tFJBjjc8zuD/93SqlkIFlEooCKGMk5RyUpQUQC1cyeBwHnzQeY/rENARARAU6afkqKXD8jG63rrEriey6sQn1mpm++a4AVSqm1Vo7NESWbPc4we56B8ff7deA3pVQf01HV5jxsJ508/u0vSaeYwoBgEaklIh7AQGC9+QARqWBaBjAM+KMEfEMxl+tnZKN1nVVJfM+FVeDPzPSlbRFwSCk13YYxOpPywDnT4yesvfEScwShlEoTkTHAT4Ar8KlS6oCIjDItnwc0BJaJSDpwEHjSbgHbQV4+IxGpBIQD5YAMEXkOaKSUSrC0rl3eSBGxwuf1BXAn4C8ikcAUpdQie7yXolKYzwzjPPxjwD7TaWCAiUqp74v4bTiS9zBOMY0HfrX2xktMmaumaZqWPyXpFJOmaZqWDzpBaJqmaRbpBKFpmqZZpBOEpmmaZpFOEJqmaZpFOkFoWhETkddE5G57x6FpudFlrppmA6abukQplWHvWDStoPQRhKblQETeFZGnzZ5PFZHnReRFEQkTY+6QUNOymqa5CuYAu4BqIrLE1Ilzn4iMM41bIiL9TY+7mDpx7jN13Cxlev2UiISKyC7TsgZF/+61kk4nCE3L2UpggNnzh4BoIBijM2YLoLVpbgeA+hjzO7QE/IGqSqkmSqmmwGLzDYuIJ7AEGGBa7gY8ZTYkRinVCpiL0b1T04qUThCalgOl1G4gUESqiEhzIB6j5cM9GN1+d2G0Ug42rXJaKbXN9PgEUFtEPhKR7kDWvl71Mbp1/mN6vhS4w2x5ZjO6nRidOzWtSJWYXkyaVgirgf5AJYwjiprA20qpT8wHmbppXs18rpSKNyWVbsBojKOPoear5LLfzA6cee6+qWnWpI8gNC13KzG6jvbHSBY/AUNN8xIgIlVN02TeRET8ARel1BpgEtAqy5DMWQvrmp4/Bvxum7egafmnv5VoWi5M3UbLAueUUheACyLSENhqFCuRCDyK8U3fXFVgsYhkfhF7Jct2k0RkCLBKRNwwWmHPs+Fb0bR80WWumqZpmkX6FJOmaZpmkU4QmqZpmkU6QWiapmkW6QShaZqmWaQThKZpmmaRThCapmmaRTpBaJqmaRb9Pyu24BoV/DLzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "als = mdf[mdf['algo'] == 'ALS']\n",
    "sns.lineplot(x='version', y='GRMSE', hue='data', data=als)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "673d66a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:32.116691Z",
     "iopub.status.busy": "2021-06-18T16:47:32.112708Z",
     "iopub.status.idle": "2021-06-18T16:47:32.266687Z",
     "shell.execute_reply": "2021-06-18T16:47:32.266687Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='version', ylabel='nDCG'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEJCAYAAACOr7BbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABEbElEQVR4nO3dd3gUVffA8e9JDxAIJUBIgCBGinSC2EWRpq/iq0iTKl2KqNhFxYIFREGRjqGoYOWHvjRRERUFIiAIWBCCBEILoQTSc39/zIJL2DTI7Kacz/Pkye7MnblnlzBnZu7ce8UYg1JKKZWdl6cDUEopVTRpglBKKeWSJgillFIuaYJQSinlkiYIpZRSLmmCUEop5ZKtCUJEOorIHyKyS0SecLFeRGSKY/1WEWnhtO4hEdkuIr+JyIciEmBnrEoppc5nW4IQEW9gKtAJaAj0EJGG2Yp1AiIdP4OBaY5tw4BRQJQxphHgDXS3K1allFIX8rFx31cBu4wxuwFEZBHQGdjhVKYzMN9YvfV+FpFgEQl1ii1QRNKBMsCBvCqsUqWKiYiIKMSPoJRSJdsvv/xy1BgT4mqdnQkiDNjn9D4OaJ2PMmHGmBgRmQj8AyQDq4wxq/KqMCIigpiYmEuLWimlShER2ZvTOjvbIMTFsuzjergsIyIVsa4u6gA1gLIi0stlJSKDRSRGRGKOHDlySQErpZT6l50JIg6o6fQ+nAtvE+VU5lZgjzHmiDEmHfgMuNZVJcaYmcaYKGNMVEiIy6skpZRSF8HOBLERiBSROiLih9XIvDRbmaVAH8fTTFcDJ4wx8Vi3lq4WkTIiIkBbYKeNsSqllMrGtjYIY0yGiIwAVmI9hTTXGLNdRIY61k8HlgG3AbuAM0B/x7r1IvIJsAnIADYDM+2KVSml1IWkJA33HRUVZbSRWiml8k9EfjHGRLlapz2plVJKuaQJQimllEt29oNQxVBmluF0WgZJKRkkpWZwKiWD06nW67PLklIzCPT1plqFAKqXt36qlvcnwNfb0+ErpQqRJogSwBhDcnomSSkZnEp1HNCdXzsO9Ek5rHM+8J9Jy7zoOCqW8aVa+QCqOxKH8+uzv4PL+GI9mKaUKuo0QXhQakbmeQfn81473p9OtQ7mSSkZnE7790DvvO50agZZ+XjWwMdLCArwoay/D+X8fQgK8KFSWT9qVSpDUIC1zHmdy9f+vpTx9+ZMWiaHTqZw8EQKB51+H3L8/m3/CY4mpV0Qg7+Pl5U4ygc4rkD8L0gq1coH4Oejdz+VcmaMIfFMOgeOJ3PgeDLxJ1I4cCKZ+OMpAEzp0bzQ69QEUUAZmVmcTs0k6dxtmHTHbZjMC15bB/pMklKcXqemOw7umaRlZuVZnwiUcxycy/n7UM5xIK9ePuDcAd354H72tat1/j5ehXb2XiHQiwqBvlxRLSjHMmkZWRw+leJIJKlWAnFKJlvjjrPyRAppGRd+D1XK+WVLJBe+Lh/oo1cjqsQ4mZJO/PF/D/rxJ5I54PgdfyKFA8eTSc32f8XXW6heIYA6VcrZEpMmCGD+T7GcSjl7dp7O6dTM814734tPTs/fLZgyft7WAdpxUC/r50N4RT+C/IOss3GnA3m57O8DrO3K+vtQxs+72B4E/Xy8CK9YhvCKZXIsY4zh+Jl06yrE6QrkbCI5cCKFzfuOc+z0hVcjgb7eVCvv/+/Vh4tEEhLkj6+3Xo0oz0pJz/z3rN/p94ETKcQ73ielZpy3jZdA1aAAQoMDaBhanrb1q1IjOJAawQGEVggkNDiAKmX98fKy7/igCQJ4bfnvnE7LxM/b69yB+uzBPSTInzpVyl54pn72wO/iNkxZP2989KCULyJCxbJ+VCzrR4PQ8jmWS83I5PDJ1H9vZ529teVIKjF7Ezl8MvWCqzIRqFLO36lNxN/RJhLo+G3d4goK8LX7o6oSKi0ji0MnUy647eN8BZB4Jv2C7aqU8yO0QiB1qpTlusurEFohgNDgQGo4flcL8vf4cUQ7ygHHz6QR6OeNv48+hVOcGWM4djrN6Qok9YKrkvgTKZxIvvA/a1k/b5dXIGfbR0IrBFClnD/eNp6tqaInM8tw5FSqy4P+2bP/I0mpZD+Mlg/woUZw4PkH/QqB564AqpUPKDJP/eXWUU6vIIDgMn6eDkEVAhGhcjl/Kpfz58oaFXIsl3y2gT1bm8jZ1+v3HOPQyRQysrX8e3sJIeX8zzWu53Rbq6y//rcqDs6eUJy73XPuCuDf2z6u/g7K+HkTWiGAGsGB1KsX4jjwB5z3u6T8DZSMT6FUAQT6eRNRpSwRVcrmWCYry5BwOu1c0ojPdiWy+8hp1v2dwKmUjAu2DfL3sdpFKgSca2gPLuOLv48X/j7e+Pt6/fvax8vx3psAX6dljnJ+3l623mMuqYwxnEzJsBp4nRp+na8E4k+kXNDo6+ftde6K8ao6lS64AggLDixVD0doglDKBS8vISTIn5AgfxqF5Xw1ciYtI9sVSOp5VyV/HTrKkaRUMvPzHHIO/Ly9zksk/j5e+Pl44e97NpnknnjOlcmjfHFKUMlpmS4P+vsdZ/7xx5M5na1Pj5dA9fLWAb9RWAXaX1ndSgBOZ/6Vy/oVyc/rKZoglLoEZfx8uCykHJeF5PyYYWaW4UxaBqkZWdZPeqbr1xmZpKb/+zol3bEsI8ux3NV21lN2CUlpOazP+1HqvPh5Z0s2BUw8F1M+JS3LOvA7P+p53HH750Qyx102+vpTIziAuiFluf7yKhfc9qlaBBp9ixtNEErZzNtLCArwJeceI/YxxpCWmXVhkskx4VyYqJzLp7jY7mRyeo7buerjcjEqBPoSWiGAsOBAWtYOPv++f4VAqlXw14dMbKAJQqkSTEQcZ+beEOD++rOynBJULokne6Ly8/Yi1OkKoIyfHqo8Qb91pZRtvLyEAC9vxyOd2tekuNEbckoppVzSBKGUUsolTRBKKaVc0gShlFLKJU0QSimlXLI1QYhIRxH5Q0R2icgTLtaLiExxrN8qIi0cy+uJyBann5MiMtrOWJVSSp3PtsdcRcQbmAq0A+KAjSKy1Bizw6lYJyDS8dMamAa0Nsb8ATRz2s9+4HO7YlVKKXUhO68grgJ2GWN2G2PSgEVA52xlOgPzjeVnIFhEQrOVaQv8bYzZa2OsSimlsrEzQYQB+5zexzmWFbRMd+DDQo9OKaVUruxMEK6GRMw+pGWuZUTED7gT+DjHSkQGi0iMiMQcOXLkogJVSil1ITsTRBxQ0+l9OHCggGU6AZuMMYdyqsQYM9MYE2WMiQoJCbnEkJVSSp1lZ4LYCESKSB3HlUB3YGm2MkuBPo6nma4GThhj4p3W90BvLymllEfY9hSTMSZDREYAKwFvYK4xZruIDHWsnw4sA24DdgFngP5ntxeRMlhPQA2xK0allFI5s3U0V2PMMqwk4LxsutNrAwzPYdszQGU741NKKZUz7UmtlFLKJU0QSimlXNIEoZRSyiVNEEoppVzSBKGUUsolTRBKKaVc0gShlFLKJU0QSimlXNIEoZRSyiVNEEoppVzSBKGUUsolTRBKKaVc0gShlFLKJU0QSimlXNIEoZRSyiVNEEoppVzSBKGUUsolTRBKKaVc0gShlFLKJU0QSimlXNIEoZRSyiVbE4SIdBSRP0Rkl4g84WK9iMgUx/qtItLCaV2wiHwiIr+LyE4RucbOWJVSSp3PtgQhIt7AVKAT0BDoISINsxXrBEQ6fgYD05zWTQZWGGPqA02BnXbFqpRS6kJ2XkFcBewyxuw2xqQBi4DO2cp0BuYby89AsIiEikh54EZgDoAxJs0Yc9zGWJVSSmVjZ4IIA/Y5vY9zLMtPmcuAI8B7IrJZRGaLSFkbY1VKKZWNnQlCXCwz+SzjA7QAphljmgOngQvaMABEZLCIxIhIzJEjRy4lXqWUUk58bNx3HFDT6X04cCCfZQwQZ4xZ71j+CTkkCGPMTGAmQFRUVPYEpLIzxvrBgMly/Di9vmC5ybbc8TuoOnh5e/jDKKUwBlJOQGBwoe/azgSxEYgUkTrAfqA70DNbmaXACBFZBLQGThhj4gFEZJ+I1DPG/AG0BXbYFukXD0JGav4OlnkeWE0uyy+2bG77KEhsWYX3nVVrDPfMhqr1C2+fSqmCST4O/3sYDu2Awd+Cb2Ch7t62BGGMyRCREcBKwBuYa4zZLiJDHeunA8uA24BdwBmgv9MuRgLvi4gfsDvbusIV+yNkpgIC4uX4cby+YFlOy7Mt8/IG8cm23Gn9ectyWp5LWVf1n1uebR95lnVVV/ay/LssPRm+fwNmtoGO46Flf2u5Usp99q6DzwbDyQNw85Pg7VfoVYgxJeeuTFRUlImJifF0GKXDqYPw+VDY/S3U/w/c+TaUqeTpqJQq+TLT4bvXrJO04NrWlXx41EXvTkR+Mca43IH2pFYXJ6g69PoM2r8Mf66EadfC7u88HZVSJdux3TC3I6ydAE17wNDvLyk55EUThLp4Xl5w7QgY9DX4lYP5neGr5yAjzdORKVWyGANbPoDpN8DRv6DLXLjrXfAPsrVaTRDq0oU2hSHfQYs+8ONbMLc9JPzt6aiUKhmSj8Mn98OSYVC9CQz7ERrd45aqNUGowuFXFu6cAl3nw7E91pnOlg8cT1cppS7K3nUw/XrY8X9wy1jo9yUE18x7u0KiCUIVroadrTOcGs2tM55PB1hnQEqp/MtMh29egujbwcsHBnwFN45xe98jTRCq8FUIh75LrTOe7Uusq4m9P3k6KqWKB5cN0S09EoomCGUPL2/rjGfAKqsxO/o2+PYVyMzwdGRKFU3ODdEJf0GX99zSEJ0bTRDKXuFRMOR7aNwVvnvVShSJez0dlVJFi3NDdGhTGPojNLrb01FpglBuEFAe7p4Bd8+GwzutRrdtn3g6KqWKhtgfrf8TOx23Zft+4daG6NxoglDu0+Re635qSD2r8XrJA5B6ytNRKeUZmenw9Ysw7z9WQ/T9qzzSEJ0bTRDKvSpGQP8VcONj8OuH1v3WuF88HZVS7nVsN8ztAN9PhKY9PdoQnRtNEMr9vH3glqeh3/+ss6i57eH7SZCV6enIlLKXMbD5fUdD9C64NxrumurRhujcaIJQnlP7Whj2gzXY39fjrKE6TmafMkSpEiI5ET7pD//3AIQ2g2Hr4Mr/ejqqXGmCUJ4VWNE6i+o8FfZvsgb92/mlp6NSqnDF/gjTroedX0DbZ61+QhXCPR1VnjRBKM8Tgea9YMhaa/jixffBF6Mh7YynI1Pq0mSmw9cvWD2iffysfkE3PFKkGqJzowlCFR1VLreGFLjuQfjlPZh5E8Rv9XRUSl2chL8dDdFvQPP7rP5AYUWvITo3miBU0eLjB+1egN5LIOUkzG4LP70LWYU4XapSdjrbED3jxn8bojtPBf9yno6swDRBqKKp7s1WI97lt8LKJ+GDeyHpsKejUip3xbAhOjeaIFTRVbYydP8Abn8DYn+wGrD/XOXpqJRyLfYHp4bo54pNQ3RuNEGook0EWg2EwWugbFXrSmL5E5Ce4unIlLKca4j+j1ND9MPFpiE6N5ogVPFQtQEM+gZaD4X106y2icO/ezoqVdol/A1z2hfrhujc2JogRKSjiPwhIrtE5AkX60VEpjjWbxWRFk7rYkVkm4hsEZEYO+NUxYRvAHR6DXp+BKcOWk85bZyjs9Yp9zMGNi+0ekQf+xvunVdsG6JzY1uCEBFvYCrQCWgI9BCRhtmKdQIiHT+DgWnZ1t9sjGlmjImyK05VDF3RwWr8q30d/O9hWHQfnE7wdFSqtEhOhI/7wf8Nh7AWjobouzwdlS3svIK4CthljNltjEkDFgGds5XpDMw3lp+BYBEJtTEmVVIEVYP7PoEOr8Cur6wG7N1rPB2VKunONkT//qXVEN3n/4p9Q3Ru7EwQYcA+p/dxjmX5LWOAVSLyi4gMzqkSERksIjEiEnPkyJFCCFsVG15ecM0DMHC1NefE/Lvgq+cgI83TkamS5ryGaP8S1RCdGzsThLhYlv1mcW5lrjPGtMC6DTVcRG50VYkxZqYxJsoYExUSEnLx0ariK7Sp9ZRTy77w41vW6LAJf3s6KlVSnNcQ7RgSpgQ1ROfGzgQRBzhPixQOZB+qM8cyxpizvw8Dn2PdslLKNb+ycMdk6LYQEmOtxsPNC7UBW1288xqid0PX+dD5nRLXEJ0bOxPERiBSROqIiB/QHViarcxSoI/jaaargRPGmHgRKSsiQQAiUhZoD/xmY6yqpGhwhzWfb1gLqxHxk/7WfL9KFcQFDdE/QsPsTagln49dOzbGZIjICGAl4A3MNcZsF5GhjvXTgWXAbcAu4AzQ37F5NeBzETkb4wfGmBV2xapKmAphVuPhj5Ph25chLgbungW1r/F0ZKo4iP0BPhsMSYfg1ufh2lElvq0hJ2JK0CV4VFSUiYnRLhPKSdwv1vzXx/fCjY9aU51623ZepIqzzHT4djz88CZUugzumW1dPZRwIvJLTl0JtCe1KtnCW1rz/TbpDt+9BtG3WW0USjlL+BvmtIMfJjk1RJf85JCXXBOEiHQWkeFO79eLyG7HTxf7w1OqEPgHwX+nwT1z4PBOq9Fx2yeejkoVBcbApgWOhug9pbIhOjd5XUE8xvkNy/5AK6ANMMymmJSyR+MuMPQHa1ynTwfA50Mh9ZSno1KekpwIH/eFpSP+7RFdChuic5NXgvAzxjh3ZPvBGJNgjPkHKGtjXErZo2Jt6LcMbnoCti62zhzjfvF0VMrd9nwP066D3/9nNUT3+T/r4QZ1nrwSREXnN8aYEU5vtVeaKp68feDmJ61EkZVhdaz7/g3IyvR0ZMpumemw+nmYdwf4Blq98K9/qNQ+pZSXvBLEehEZlH2hiAwBNtgTklJuUvsa65ZTgzusYRTmd4YT+z0dlbLLuYboN6FFb6shukZzT0dVpOX6mKuIVAWWAKnAJsfillhtEXcZYw7ZHWBB6GOu6qIYA1s+gGWPgrcv3Pk2NLzT01GpwnK2R/Tyxx3/vlO0rcFJbo+55vpAuGOYi2tF5BbgSsfi/xljvinkGJXyHBFrspdaV1uN1x/1hpb9oMN4awgPVXydOQZfjoYd/wcRN8B/Z2hbQwHkmiBEpBVQxRizHPjGafkdwAFjjLbuqZKjcl24fxWsGQ8/vAV711mPxoY28XRk6mLs+R4+H+LoET0Orh2pbQ0FlFcbxARgp4vlOx3rlCpZfPwcT7UssR6Bnd0WfpoKWVmejkzlV0aai4bo0ZocLkJeCaKyMSY2+0JjzC6gsi0RKVUUXNbGGvTv8naw8il4vwucKlJNbsqVo7usp9J+eBNa9NGG6EuUV4IIzGWd3pxVJVvZytD9fbh9knW7adq18OcqT0elXDEGNs2HGTdYQ6l0XWA1Rmsb0iXJK0GsFpGXxTGs6lkiMg6nNgmlSiwRaDXAmpAoqDp8cK/1NEx6iqcjU2edOQYf9YGlIyE8ytEjWp9CKwx5DWv5CDAb2CUiWxzLmmHN9TDQvrCUKmKq1oeBX8PX4+Dnd60G0C5zrGE7lOfsWQufDYHTR6DdC3DNSGsqWlUo8jXct4hcxr+PuW43xuy2NaqLpP0glFv89RUsGWY1Yrd/CVoNtK40lPtkpFlzffw42Xr67J7Z2tZwkS5puG8R8cFKDvUdPw0dy5QqnSLbWbcxIq6HZWNgUU84neDpqEqPo7usHtE/vqUN0TbLa7jvGsB2rFtNNYAw4FFgu2OdUqVTuarQ82Po+CrsWm01YO9e4+moSjbnhujje635x7Uh2lZ5DbURDWwxxryVbfkooKUxpq+t0RWQ3mJSHnFwG3wyAI7+aXXGumWs1Z9CFZ4zx+CLB2HnUqhzo9Ujuryeo55ljEEu8jbnRQ+1AVxtjOnnIpgpIvLHRUVTBI38eiSpmakY/k2W516bbO+dXjsnV1fLLtiX0/rc6sqzDhf7c1VfQbctSPl2tdsxovkIvEQbBKne2HrKadXTsG6K1XB6zxyocrmnIysZtCE6V4t+X8RPB37ijTZv4ONVuHf/89pbci7rzhRmIJ50JuMMqZmpAAhWFnbOxmeXOTu7/lx5xFom55c/99ppF87b5LQ/521c7c9V+Uupy9XZR07fxfHU48zaNov9Sft56bqX8PX2vWDbUsevDPznTajb1pqAZsYN0Ol1a/pKbcC+ONkbonushhrNPB1VkWGM4e3NbzNr2yzahLchIyvD7Qmigojc7WK5AOXz2rmIdAQmA97AbGPMq9nWi2P9bVgJp58xZpPTem8gBthvjPlPXvVdrDkd5ti16xLJGMOc3+YwedNkElMSefPmNynrq/eBAWjwH2t2ss8GW4ni76+txBFYMe9t1b+O7rIGTozfogMnupCRlcELP73A57s+557Ie3jm6mcKPTlA3gniO+AOp/dn7zGIY12OHAf3qUA7IA7YKCJLjTE7nIp1AiIdP62BaY7fZz2INe5TnslIuY+IMLDxQKoEVuH5dc/Tf0V/3r31XaoEVvF0aEVD+RrWDGXrpsA3L8G+jXD1UJCzYwEZq8H17GtwvM/P67Pb4/TauH6d733l9Zoctr/U/braHjCZ1uirPv5WQ3QD50OQSs5IZsx3Y1gbt5ahTYfyQNMHLrr9IS95DffdH0BEAoB7gAinbfLqQHEVsOtsnwkRWQR0BpwTRGdgvrFubv8sIsEiEmqMiReRcOB24GXg4QJ9KuUWd11+F5UCKjHmuzH0XtabGe1mUKt8LU+HVTR4eVszldW5ET4dBKue8WAw4nSbK9trcLzPz2tX+3Kq45L35fS6zk3wn0naEJ1NYkoiI74ZwW9Hf2Ps1WPpWq+rrfXl95pkCXAca9Kg/I4xEAY4z2cdx/lXBzmVCQPigbeAx4CgfNanPODG8BuZ3X42w78eTu/lvXn31ne5svKVeW9YWoS1hBEbIfWkY0EhHDzzOsBrm0eJtD9pP0O/GsqBpANMumkSbWu3tb3O/CaIcGNMxwLu29VfafarDpdlROQ/wGFjzC8i0ibXSkQGA4MBatXSs1dPaBLShPmd5jP0q6Hcv+J+3rz5Ta6tca2nwyo6vLy1DUJdkj+O/cGw1cNIyUxhVvtZtKjWwi315vdZsXUi0riA+44Dajq9DwcO5LPMdcCdIhILLAJuEZGFrioxxsw0xkQZY6JCQkIKGKIqLHUq1GHBbQsIDwpn+NfD+d/u/3k6JKVKhA3xG+i3oh9e4sX8jvPdlhwg/wnieuAXEflDRLaKyDYR2ZrHNhuBSBGpIyJ+QHdgabYyS4E+YrkaOGGMiTfGPGmMCTfGRDi2+8YY0yv/H0t5QtUyVYnuGE3zqs154vsnmLd9nqdDUqpYWxG7gqGrh1KtTDUW3raQyyu6t29Nfm8xdSrojo0xGSIyAliJ9ZjrXGPMdhEZ6lg/HViG9YjrLqzHXPsXtB5VtAT5BTHt1mk8+f2TTIyZyJEzR3g46mHtUKdUAb2/831e2/Aazas2Z8otU6jgX8HtMeRrNNfiQofaKDoyszJ5dcOrLPpjEbdfdjsvXvuidqhTKh+MMUzZPIXZ22ZzS81beO3G1wjwCbCtvksZakOpi+Lt5c1TrZ+iapmqTNk8xepQ1+ZNyviW8XRoShVZ6VnpPL/ueZb+vZR7r7iXp1s/jbcH59LW635lGxFhUJNBvHDtC6yPX8/9K+8nIVmHxVbKlTPpZxj1zSiW/r2U4c2GM/bqsR5NDqAJQrnBfyP/y+SbJ/P38b/pvbw3+07uy3sjpUqRYynHGLByAOsOrOP5a55naNOhtvWOLghNEMotbqp5E7Paz+Jk2kl6Le/FjoQdeW+kVCmw79Q++izvw1/H/+KtNm9xzxX3eDqkczRBKLdpVrUZ8zvOx9/bn/4r+vPTgZ88HZJSHrUzYSe9l/UmMSWR2e1nc3Otmz0d0nk0QSi3uiz4MhZ0WkBYUBgPfP0Ay3Yv83RISnnEz/E/039lf3y9fVnQaQHNqjbzdEgX0ASh3K5a2WpEd4ymaUhTHv/+ceZvn+/pkJRyq+V7ljNs9TBCy4aysNNCLgu+zNMhuaQJQnlEeb/yzGg3g1tr3cqEmAlMiplElsnydFhK2W7BjgU8tvYxmoY0ZV6neVQrW83TIeVIE4TyGH9vfybeNJFu9brx3vb3eOaHZ0jPSvd0WErZIstkMSlmEq9vfJ12tdsxo90MyvsV7aluSnxHufT0dOLi4khJye8o5aVLQEAA4eHh+Pp6ppezt5c3T7d+mqplqvL25rc5lnKMSW0maYc6VaKkZ6bz7Lpn+XL3l3Sr140nr3rS430c8qPEJ4i4uDiCgoKIiIgoEs8VFyXGGBISEoiLi6NOnToei0NEGNxkMFUCqzDup3EMWDmAqbdOpVJAJY/FpFRhOZ1+mofXPMy6A+sY2XwkgxoPKjbHohJ/iyklJYXKlSsXm38QdxIRKleuXGSuru6OvJvJN0/mr+N/0XtZb/ad0g51qng7mnyU+1fez/r49bxw7QsMbjK4WB2LSnyCAIrVP4i7FbXvpk3NNsxuP5vjqcfpvaw3OxN2ejokpS7KvpNWB7jdx3cz5ZYp/Dfyv54OqcBKRYLwhOeff56JEyfmuH7JkiXs2KG9iV1pVrUZCzotwNfbl/4r+7M+fr2nQ1KqQLYnbKfX8l6cSjvF7A6zuTH8Rk+HdFE0QXiIJojcne1QF1o2lKGrh7JizwpPh6RUvqw7sI77V9xPgHcA8zvNp2lIU0+HdNE0QRSil19+mXr16nHrrbfyxx9/ADBr1ixatWpF06ZNueeeezhz5gzr1q1j6dKlPProozRr1oy///7bZbnSrnrZ6kR3jKZJlSY8uvZRFu5wOeusUkXGl7u/ZPjq4dQMqsmC2xZQp4LnHv4oDJogCskvv/zCokWL2Lx5M5999hkbN24E4O6772bjxo38+uuvNGjQgDlz5nDttddy5513MmHCBLZs2ULdunVdllNQwb8CM9rNoG2ttry28TXe/OVNStIkV6rkmLd9Hk9+/yTNqzXnvY7vUbVMVbfVbTIzbdlviX/MNT9Sd+3CZOX3oOO63DeffsodN96IV1wcfsBtN9xA+pEjbFq+nOffeosTp06RdPo07a6/npQOHcg8eZK0AwdI+fNPADZt2OCynDukHzrErpGjCrRN0K23UnXMI4iP/X9CAT4BvHHTG7y8/mXm/jaXo8lHef7a5/H10hnqlOdlmSwmxkxkwY4FtK/dnldueAU/bz+31G2MIWH2bM78vJ6a06chhdyfSRMEIAEByCWelYqvL16+vngFBlrvfXwQX18GPfUUn8yaRZOGDZn/8ces/eknvAIDEW9vvPz8zpXPqVwOtRUgsHwU8fMjsEXzfD/RlHniJMeio0n96y/C3pyEd3n7e4N6e3kz9uqxVC1TlalbpnIs5Rhv3PSGdqhTHpWWmcYzPz7D8j3L6Vm/J49f9bjb5l/PSkvj4LPPcWLJEsrf1gmTlVWQI0O+aIIA/MLDL3kft3TuTL9+/Xj6lVfIyMhg2Zo1DBkyhKQzZ6jVvDlSsSKLly8nLCwMv5o1qVC9Osn+/vjVrAmQYzl38Dl1irDXXy/QNsc/+YT458cR270HNadPw69WLZui+5eIMLTpUKoEVuHFn19k4KqBvNP2He1QpzwiKS2J0WtGsz5+PaNbjOb+Rve77bHxjMRE4kaOJDnmF6oMH06VEcNtqVvbIApJixYt6NatG82aNeOee+7hhhtuAODFF1+kdevWtGvXjvr1658r3717dyZMmEDz5s35+++/cyxXVAV36UKtOXPITEggtms3zsTEuK3uLld04c02b/Jn4p/0Wd6HuFNxbqtbKfi3A1zMwRheuu4lBjQe4LbkkLprF7H3diVl22/UeGMiISNH2Fe3Mca2H6Aj8AewC3jCxXoBpjjWbwVaOJYHABuAX4HtwLj81NeyZUuT3Y4dOy5Yps53Kd9Ramys2dWxk9nRqLFJ/PSzQowqb5sObTLXfnCtabO4jfk94Xe31q1Kr9gTsabDJx1Mq4WtzNp9a91a96m135vfW0aZP6673pzZsqVQ9gnEmByOqbZdQYiINzAV6AQ0BHqISMNsxToBkY6fwcA0x/JU4BZjTFOgGdBRRK62K1Z18fxq1yZi0YeUiWpJ/FNPcfiNNzBZ7hm2u3nV5szrOA9v8abfin5siN/glnpV6fXb0d/ovaw3Z9LPMKf9HG4Iv8FtdR9b+D77hgzBNzycOh8tJrCp/f0r7LzFdBWwyxiz2xiTBiwCOmcr0xmY70hkPwPBIhLqeJ/kKOPr+NFnG4so7woVqDVzJsHdupEwazZxo0aR5aZ+HJdXvJyFty2kWplqVoe6WO1Qp+zxw/4fuH/l/ZTxLcP8TvNpHNLYLfWajAwOvvACh156iXJt2hDx/kJ8a9RwS912JogwwHm0tTjHsnyVERFvEdkCHAa+MsboeAtFmPj6Uv3556j21JMkffMtsb16kX7woFvqrl62OvM6zaNxlcY89t1jvL/zfbfUq0qPpX8vZeTXI6ldvjYLb1tIRIUIt9SbefIk+wYPIfGDD6k04H7C356CV9mybqkb7E0QrlpNsl8F5FjGGJNpjGkGhANXiUgjl5WIDBaRGBGJOXLkyKXEqy6RiFCpTx9qTp9G+t5/iL23K8nbfnNL3Wc71LWp2YZXN7zK5E2TtUOdumTGGOZsm8PTPzxNy+otea/De1QJrOKWutP27iW2ew9Ob9xI6MsvUe3RRxFv984hYWeCiAOcn9MMBw4UtIwx5jiwBqvB+wLGmJnGmChjTFRISMglhqwKQ7kbb6T2hx8gvr7s7d2bkytWuqXeAJ8AJrWZRJcrujB722zG/jhWZ6hTFy3LZPH6xtd5a9NbdKrTiWltp1HOr5xb6j69YQOxXbuRmZBArTmzCb7nHrfUm52dCWIjECkidUTED+gOLM1WZinQRyxXAyeMMfEiEiIiwQAiEgjcCvxuY6yqkAVccQURH39EQIMG7B89mqPTprnljN7Hy4dnr36WB5o+wP/9/X88+M2DnEnXca1UwaRlpvHY2sdYuHMhvRv25tUbXsXX2z09949/+hn/DBiId+XKRHy0mLJXXeWWel2xLUEYYzKAEcBKYCfwkTFmu4gMFZGhjmLLgN1Yj7nOAh5wLA8FvhWRrViJ5itjzJd2xepp0dHRjBgxAoC1a9fSokULfHx8+OSTT84rN2/ePCIjI4mMjGTevHnnlu/Zs4fWrVsTGRlJt27dSEtLA/IectxuPpUrUyv6PcrfcQdHJk/hwGOPk5Waanu9IsKwZsMYe/VYfjzwI4NWDSIxJdH2elXJcCrtFMNWD2Nl7ErGRI3hsVaPuaV3tMnM5NDrE4h/+mnKtmpFxKIP8atd2/Z6c2PrpzbGLDPGXGGMqWuMedmxbLoxZrrjtTHGDHesb2yMiXEs32qMaW6MaWKMaWSMecHOOIuSWrVqER0dTc+ePc9bfuzYMcaNG8f69evZsGED48aNIzHROug9/vjjPPTQQ/z1119UrFixSA305+XvT43XXyNk9IOc/OIL/unXn4yEBLfU3bVeVya1mcTvx36nz/I+7E/a75Z6VfF15MwR+q/oz6ZDmxh//Xj6XtnXLfVmnT5N3MhRHJs7l4o9e1Bz5gy3DGGTF+1JbbPY2Fjq16/PwIEDadSoEffddx+rV6/muuuuIzIykg0bzn92PyIigiZNmuDldf4/zcqVK2nXrh2VKlWiYsWKtGvXjhUrVmCM4ZtvvqFLly4A9O3blyVLllwQx6xZs+jUqRPJycm2fdaciAhVhg4l7K23SNm5k9iu3c4NUmi3trXaMqv9LBJSEui9rDd/HPvDLfWq4mfPiT30WtaLf079w9S2U7mj7h1uqTf9wAFi7+tF0po1VHvmGao/+6xbBsHMj6IRhZuM+2I7Ow6cLNR9NqxRnufuuDLXMrt27eLjjz9m5syZtGrVig8++IAffviBpUuXMn78eO66664869m/fz81ncZmCg8PZ//+/SQkJBAcHIyP4w/q7HJn77zzDqtWrWLJkiX4+/sX/EMWkvIdO+AbFkbcAw+wt0dPwia9QbmbbrK93hbVWjCv4zyGrh5KvxX9mHLLFFpVb2V7var42HpkK8O/Ho6XePFeh/e4skru/6cLS/Kvv7Jv+AhMSgo1Z8yg3A3Xu6Xe/NIrCDeoU6cOjRs3xsvLiyuvvJK2bdsiIjRu3JjY2Nh87cNVA6+I5Lj8rAULFrB8+XI+/fRTjyaHswIbNyLi44/wrVWLfcMe4Nj8+W5pvI6sGMn7t71P1TJVGfLVEFbFrrK9TlU8rI1by4CVAwjyC2JBpwVuSw4n/vc/9vbpi1dAABGLPixyyQFK2RVEXmf6dnE+MHt5eZ177+XlRUZGRr72ER4ezpo1a869j4uLo02bNlSpUoXjx4+TkZGBj48PcXFx1HDqZdmoUSO2bNlCXFwcdeoUjdmtfKtXJ2LhAvY//jiHxr9C6u7dVH/66UIfyz676mWrM7/TfEZ8PYIx343hyZQn6VG/h611qqLt878+Z9xP46hXqR5T2051Sx8HYwxH35nK0alTCWzZkvC3p+BTqWiOSKxXEMVEhw4dWLVqFYmJiSQmJrJq1So6dOiAiHDzzTefe+Jp3rx5dO7874gmzZs3Z8aMGdx5550cOJC9G4rneJUtS/iUKVQeNJDjixbzz+DBZJ44YXu9FfwrMKv9LG6qeRPj149nyqYp2qGuFDLGMGvrLJ5d9yytQ1szt8NctySHrJQUDjwyhqNTp1Lhrruo9d7cIpscQBNEkbNx40bCw8P5+OOPGTJkCFdeaV31VKpUibFjx9KqVStatWrFs88+SyXHH9Zrr73GpEmTuPzyy0lISGDAgAHn7fP6669n4sSJ3H777Rw9etTtnykn4uVF1UceIXT8eM7E/EJs9x6k7d1re70BPgG82eZN7om8h1nbZvHcuufIyMrflZwq/jKzMq2Tg81TuP2y23nnlnco62v/8BUZR46wt29fTi5fTtUxjxD6yni8/Nwz89zFkpJ09hQVFWViss1LsHPnTho0aOChiIqHovAdndm4kbiRo8AYwqZMoWxr+zsHGWN499d3mf7rdG4Kv4kJN00g0CenWfxUSZCamcqT3z/JV3u/ot+V/Xio5UNu6eOQ8vvv7Bv2AJnHjxM24XWCbr3V9jrzS0R+McZEuVqnVxCqSCjTqhURHy3Gu3Jl/hkwgOPZOgnaQUQY3mw4Y68ey9q4tQxcNZDjKcdtr1d5xsm0kwz5aghf7f2KMVFjeCTqEbckh1PffENsz/vAGCLeX1ikkkNeNEGoIsOvVi0iFn1I2datiX9mLIden4DJzLS93nMd6hJ+p/fy3hxIKjptNapwHDp9iL7L+/LrkV957YbX3NIBzhhDwpw5xA0fgX/dukR8tJiAhtmnxCnaNEGoIsW7fHlqzphOxZ49OTZ3LnEjR5F1+rTt9d5a+1ZmtJtBQrLVoe7PRPd05FP22318N72W9+JA0gHebfsut112m+11mrQ04p9+hsMTJhLUsQO158/Dt2pV2+stbJogVJEjPj5Uf3Ys1Z55hqQ1a4i9rxfp8fG21xtVPYroTtEg0G95P2IOum+ebWWPLYe30GdFH9Iz04nuGM01Na6xvc6MxET+uX8AJz77jCoPPEDYG2/gFVg827Y0Qagiq1Kv+6g5YzrpcXHs6dqV5K1bba/ziopXsLDTQqqUqXLufrUqntbsW8OgVYOo4FeBBbctoEFl+x/ESN29m9hu3UneupUaEyYQMmok4lV8D7PFN3JVKpS74QYiFn2Il38Ae3v34eSyZbbXGVoulPkd59OgcgMeWfMIi39fbHudqnB9+uenPPjtg1wefDnzO82nZlDNvDe6REk//khst+5knT5NrXnRVLjjP7bXaTdNEEVAfof7Lq38L7/cauBr1Ij9Dz/CkalTbe/cFhwQzKz2s7gx/EZeWv8Sb29+WzvUFQPGGKb/Op3nf3qea2pcw5wOc6gcWNn2eo998AH7Bg/BNzSUOh8tpkzz5rbX6Q6aIIqYnIb7Lu18KlWi1ntzqdC5M0fffocDYx61fW6JQJ9A3rr5Lf57+X+ZuXUm434apx3qirDMrExe+vklpm6Zyp117+TtW96mjG8ZW+s0GRkcfPElDr3wIuVuuIHaH3yAb1iYrXW6U6kai8kTYmNj6dixI9dffz0///wzTZs2pX///jz33HMcPnyY999//7zyERERABcM971mzRqee+45qlWrxpYtW7j77rtp3LgxkydPJjk5mSVLllC3bl13fSyP8PLzI/TVV/CrW5cjkyaRHhdH+NR38Kli3xAJPl4+jLt2HCFlQpi5dSYJyQm8ftPr2qGuiEnJSOHxtY/zzb5vGNBoAA+2ePC8QSvtkHnqFPsfepjTP/xApf79qTrmEbfPGW230pUglj8BB7cV7j6rN4ZOr+ZapDCG+wb49ddf2blzJ5UqVeKyyy5j4MCBbNiwgcmTJ/P222/z1ltvXfrnKeJEhCqDB+EXUZsDjz3Onq5dqTltGgH16tla58jmIwkJDGH8+vEMWjWId255h+CAYNvqVPl3IvUEo74ZxebDm3niqie4r8F9tteZtm8f+4YOI23vXqq/+AIV773X9jo9QW8xuUFhDPcN0KpVK0JDQ/H396du3bq0b98eoMD7KQnKt29P7fcXQkYme3v05NS339peZ/f63XmjzRvsSNhB3xV9iU+y/9FblbuDpw/Sb0U/th3dxus3ve6W5HAmJobYe7uScfQotWbPLrHJAUrbFUQeZ/p2KYzhvgtzPyVF4JVXEvHxR8Q9MJy4B4ZT9bHHqNSvr623FtrVbkdwu2BGfTOKXst7Mf3W6URWjLStPpWzXYm7GLp6KEnpSUy/dTpXhdo/ftfxzz4n/rnn8AsLo+b0afg5bgmXVHoFoYo132rVqL1wAUHt2nH4tdc4+OxzmPR0W+tsVb0V0R2jwUDfFX21Q50HbDq0iT4r+pBpMonuGG17cjBZWRx+4w3in3qKMlEtiVi8qMQnB9AEUeTkNNy3yplXYCBhb71J5SFDOP7xx/wzaDCZx4/bWme9SvVYcNsCKgdUZshXQ/h679e21qf+9fU/XzP4q8FUDqjMwtsWUr9SfVvryzp9mrhRo0iYNZvg7t2oNXMm3hUq2FpnkWGMse0H6Aj8AewCnnCxXoApjvVbgRaO5TWBb4GdwHbgwfzU17JlS5Pdjh07LlimzleSvqPjS5aYnY0am13tO5iU3bttry8xOdH0/F9P02ReE7P498W211faLf59sWkyr4np+WVPcyz5mO31pcXHm7/v+q/Z0aChSZg332RlZdlep7sBMSaHY6ptVxAi4g1MBToBDYEeIpJ9KMNOQKTjZzAwzbE8A3jEGNMAuBoY7mJbpS5QoXNnakW/R+bJk8R278Hpn3+2tb7ggGBmtZvF9WHX8+LPLzJ1i/2d+EojYwxTt0zlxZ9f5Loa1zGr/SwqBlS0tc7kbdvYc++9pP/zDzWnT6NSn962Pzpb1Nh5i+kqYJcxZrcxJg1YBHTOVqYzMN+RyH4GgkUk1BgTb4zZBGCMOYV1JVFyep8oW5Vp2ZKIjz/CJ6QK/wwcROJHH9lbn28Z3rr5Le66/C6m/zqdF35+QTvUFaKMrAzG/TSO6b9O567L72LyLZNt7wB3cvly9vbqjZefPxGLPqTcjTfaWl9RZWeCCAP2Ob2P48KDfJ5lRCQCaA6sL/wQVUnlFx5OxIcfUvaaazj47HMcevU1W+eW8PXy5YVrX2BQ40F88ucnPLzmYVIyUmyrr7RIzkjmoTUP8elfnzKo8SBeuPYFfL18bavPGMORd99l/0MPE+B4Ss4/svQ+pWZngnB1LZb92jvXMiJSDvgUGG2MOemyEpHBIhIjIjFHjhy56GBVyeMdFETNae9SsXdvjkVHEzd8BJlJ9s0tISKMajGKJ6968txIoidST9hWX0l3IvUEg1cN5rt93/FU66cY1WKUrbd4slJTOfDoYxyd8va5W5U+jnnfSys7E0QcVmPzWeFA9qm6ciwjIr5YyeF9Y8xnOVVijJlpjIkyxkSFhIQUSuCq5BAfH6o//RTVnh1L0vffs7dnT9L377e1zp4NejLhpglsT9hO3+V9OXj6oK31lUTxSfH0Wd6H7QnbmXjTRHrU72FrfRlHj/JPn76c/PJLQh56iNBXX8HLz8/WOosDOzvKbQQiRaQOsB/oDmQfgW4pMEJEFgGtgRPGmHixThPmADuNMZNsjFGVEpV69sSvdm32j36IPV27UXPqOwQ2a2ZbfR0iOlDRvyIPfvsg9y27j0daPoK/t7/LsuaCC+vclwO5NoTnul0u63JfVbix5La/9Mx03tn8DskZycxoN4NW1VvlHFghSPnjD/YNG0bmsUTCpkymvGOEAmVjgjDGZIjICGAl4A3MNcZsF5GhjvXTgWXAbViPuZ4B+js2vw7oDWwTkS2OZU8ZY+yfDMADoqOjiYmJ4Z133mHt2rWMHj2arVu3smjRIrp06XKu3Lx583jppZcAeOaZZ+jb1/55dUuSctddR8TiRewbOoy9ffoSOn48Ff5zu231XRV6FdEdoxm2ehiPf/+4bfWURFUDqxLdKZorKl5haz2nvvmWA2PG4FWuHLXfX0ig9js6j61DbTgO6MuyLZvu9NoAw11s9wOu2ydKvLPDfU+cOPG85ceOHWPcuHHExMQgIrRs2ZI777yTihXtfdSvpPG/7DIiFi9i/8hRHBgzhrTdu6kycoRt97brVarH0ruWsj/p4m5r5RaX5PJfJNd1F7nP3FcVbixVy1S1dcRcYwzHoudx+PXXCWjYkPB338W3WvGbM9pupWssJg8orOG+V65cSbt27ajkaDRr164dK1asoEePHkRERNCzZ0++/fZb0tPTmTlzJk8++SS7du3i0UcfZejQoW75rMWFT8WK1Jo7h/jnx3H03XdJi91D6PjxeAUE2FJfOb9y1Ktk32izqmBMWhoHX3yR4x9/QlCHDtR49ZViO2e03UpVgnhtw2v8fuz3Qt1n/Ur1efyq3G8fFMZw3/v376dmzX/b88PDw9nv1Nhas2ZNfvrpJx566CH69evHjz/+SEpKCldeeaUmCBfEz4/Ql1/Cv+5lHJ74Bmlx+wl/5218q+pZZEmWkZjI/gdHc2bDBioPHULIqFHFes5ou+k34waFMdy3q4ZA58vzO++8E7CG/m7dujVBQUGEhIQQEBDAcZvHJSquRITKAwYQ/s7bpP71F7Fdu5Gyc6enw1I2Sd29h9ju3UnevJkar79G1dGjNTnkoVRdQeR1pm+XwhimOzw8nDVr1px7HxcXR5s2bS6ow3n/Ba2jtApq25aI9xey74HhxN7Xi7CJEwi65RZPh6UK0el164gb/RDi40OtefMo06JkzBltN02fxUSHDh1YtWoViYmJJCYmsmrVKjp06ODpsEqMgIYNifhoMf516xI3fAQJc+bqmEolROKiRfwzaDC+1aoR8dFHmhwKQBNEEZPTcN+VKlVi7NixtGrVilatWvHss8+ea7BWhcO3alVqz59HUIcOHJ4wgfixYzFpaZ4OS10kk5HBwZfHc/D5cZS7/npqf/gBfuE6pFtBSEk6S4qKijIxMedP3rJz504aNGjgoYiKB/2Ozmeysjj6zjscfXcaZVq1ImzKZHz0ceJiJfPUKfY/8gin135Ppb59qfrYo4i3t6fDKpJE5BdjTJSrdXoFoVQ24uVFyKhR1JjwOsm//kpst+6k7t7t6bBUPqXFxbG3Z09Or/uJ6uPGUe3JJzQ5XCRNEErloMIdd1BrXjRZp08T2607p9et83RIKg9nNm0i9t6upB8+Qq3Zs6jYraunQyrWNEEolYsyzZsTsXgxvtWr88+gwSQuWuTpkFQOTvzf//FP3354ly9PxKIPKXv11Z4OqdjTBKFUHvzCw6j94QeUu/56Dj4/joPjx9s6t4QqGJOVxeFJb3Lg8ScIbNmSiMWL8K9Tx9NhlQiaIJTKB+9y5Qh/dyqV+vYlcf4Ca/TPpCRPh1XqZZ05w/4HR5MwcybBXbtSa9ZMvIODPR1WiaEJQql8Em9vqj35BNXHjeP0up/Y26MHaXFxng6r1Eo/dIi9vXpz6uuvHf8uzyO+9s02VxppgigCoqOjGTFiBACTJk2iYcOGNGnShLZt27J3795z5ebNm0dkZCSRkZHMmzfPU+GWehW7WWeq6YcOE9u1G2c2bfZ0SKVO8rbfiO1yL2l79567srNztrnSShNEEdO8eXNiYmLYunUrXbp04bHHHgP+He57/fr1bNiwgXHjxpGYmOjhaEuvstdcQ8SiRXgFleOfvn058cUXng6p1Di5YiV7e/dGfH2p/cEHBDkNOaMKlyYIm8XGxlK/fn0GDhxIo0aNuO+++1i9ejXXXXcdkZGRbNiw4bzyN998M2XKlAHg6quvJs5xC8N5uO+KFSueG+4brCHCn3rqKa655hqioqLYtGkTHTp0oG7dukyfPh1lD//L6hCxaBGBzZtz4NHHODx5MiYry9NhlVjGGI5On87+0aMJqF+fiI8/IqCevRMKlXalarC+g+PHk7qzcIf79m9Qn+pPPZVrmYsd7nvOnDl06tQJ0OG+iyqfihWpNXsWB198kYRp00nbvUfnF7BBVmoq8c+M5eQXX1D+jjsIfelFvPxdT+GqCk+pShCecna4byDfw30vXLiQmJgYvvvuO6Bgw30nJSURFBREUFDQueG+g/XJDtuInx/VX3gBv8vqcvj119m7fz/hU6fqDGWFJCMhgbgRI0nevJmQ0Q9SecgQbW9wk1KVIPI607dLQYf7Xr16NS+//DLffffdubI63HfRJiJU7t8Pv9q12T9mDLFdu1Jz2rsENGzo6dCKtZQ//yRu6DAyjh0j7K23KN9RRzB2J22DKGI2b97MkCFDWLp0KVWdZjfT4b6Lh6Bbbibig/fBy4vY+3pxavVqT4dUbCV99x17e/TEZGRQe8ECTQ4eoAmiiHn00UdJSkri3nvvpVmzZuduHelw38VHQP361PloMf5XRBI3chRHZ83SuSUKwBhDQnQ0+4Y9gF/t2kR8/BGBjRt5OqxSydbhvkWkIzAZ8AZmG2NezbZeHOtvA84A/Ywxmxzr5gL/AQ4bY/L116HDfV8c/Y7skZWSQvxTT3Ny2TJ8a9Uqmp24imDiMhkZpP/zD0Ht2lHjtVfxcjzVp+yR23DftrVBiIg3MBVoB8QBG0VkqTFmh1OxTkCk46c1MM3xGyAaeAeYb1eMStnJKyCAGm9MJLBpE85s3uLpcHJWBNt7K3brRqX+/XTOaA+zs5H6KmCXMWY3gIgsAjoDzgmiMzDfWJcxP4tIsIiEGmPijTFrRSTCxviUsp2IUKlvXyr17evpUJQqMDvTcxiwz+l9nGNZQcsopZTyADsThKsL1+w3PPNTJvdKRAaLSIyIxBw5csRlGW0gzJl+N0qpnNiZIOKAmk7vw4EDF1EmV8aYmcaYKGNMVEhIyAXrAwICSEhI0AOhC8YYEhISCAgI8HQoSqkiyM42iI1ApIjUAfYD3YGe2cosBUY42idaAyeMMfGFGUR4eDhxcXHkdHVR2gUEBBAeHu7pMJRSRZBtCcIYkyEiI4CVWI+5zjXGbBeRoY7104FlWI+47sJ6zLX/2e1F5EOgDVBFROKA54wxcwoah6+vL3V0dimllCowW/tBuJurfhBKKaVylls/CH3IWCmllEuaIJRSSrlUom4xicgRYG+eBV2rAhwtxHBKOv2+Cka/r4LR76tgLuX7qm2MufARUEpYgrgUIhKT0304dSH9vgpGv6+C0e+rYOz6vvQWk1JKKZc0QSillHJJE8S/Zno6gGJGv6+C0e+rYPT7Khhbvi9tg1BKKeWSXkEopZRyqVQlCBHpKCJ/iMguEXnCxfqKIvK5iGwVkQ0iUurmOczHd1RfRH4SkVQRGVOQbUuiS/y+5orIYRH5zX0Re97FfmciUlNEvhWRnSKyXUQedG/kxYeIRInIlEveT2m5xeSY4e5PnGa4A3o4z3AnIhOAJGPMOBGpD0w1xrT1SMAekM/vqCpQG7gLSDTGTMzvtiXNpXxfjnU3AklYk2aVipORS/wbCwVCjTGbRCQI+AW4qyT/jXlaabqCODfDnTEmDTg7w52zhsDXAMaY34EIEanm3jA9Ks/vyBhz2BizEUgv6LYl0KV8Xxhj1gLH3BJp0XHR35ljpslNjtengJ2U4AnGRCRCRH4Xkdki8puIvC8it4rIjyLyl4hc5fhZJyKbHb/rObZtIyJfOl4/77haXSMiu0VkVH5jKE0JIj+z1/0K3A0gIldhncWUprGwL2WGv9I4O2Bp/MyXqlC+M8d0xM2B9YUTVpF1OTAZaALUx5oy4XpgDPAU8DtwozGmOfAsMD6H/dQHOmAl6OdExDc/lds5H0RRk5/Z614FJovIFmAbsBnIsDmuouRSZvi75NkBi6HS+JkvVWHMIlkO+BQYbYw5WShRFV17jDHbAERkO/C1McaIyDYgAqgAzBORSKzvMacD//+MMalAqogcBqphJedclaYEkefsdY4/tv4AIiLAHsdPaXEpM/xd8uyAxVBp/MyX6pK+M8eZ76fA+8aYzwo5tqIo1el1ltP7LKzj94vAt8aY/zquqtbkYz+Z5PPYX5puMZ2b4U5E/LBmuFvqXEBEgh3rAAYCa0vBGYqzPL8jm7YtrkrjZ75UF/2dOU7a5gA7jTGTbIyxOKmANWMnQL/C3nmpuYLI5wx3DYD5IpIJ7AAGeCxgD8jPdyQi1YEYoDyQJSKjgYbGmJOutvXIB3GTQvi+CmXWxOLkUr4zrPvwvYFtjtvAAE8ZY5a5+WMUJa9j3WJ6GPimsHdeah5zVUopVTCl6RaTUkqpAtAEoZRSyiVNEEoppVzSBKGUUsolTRBKKaVc0gShlJuJyAsicqun41AqL/qYq1I2cHTqEmNMlqdjUepi6RWEUrkQkddE5AGn98+LyCMi8qiIbBRr7pBxjnURjrkK3gU2ATVFJNoxEuc2EXnIUS5aRLo4Xrd1jMS5zTHipr9jeayIjBORTY519d3/6VVppwlCqdwtAro5ve8KHAEisUbGbAa0dMztAFAPa36H5kAVIMwY08gY0xh4z3nHIhIARAPdHOt9gGFORY4aY1oA07BG71TKrTRBKJULY8xmoKqI1BCRpkAi1pAP7bFG+92ENZRypGOTvcaYnx2vdwOXicjbItIRyD6uVz2s0Tr/dLyfB9zotP7sYHS/YI3cqZRblZqxmJS6BJ8AXYDqWFcUEcArxpgZzoUco2mePvveGJPoSCodgOFYVx/3O2+SR71nR+DM9+ibShUmvYJQKm+LsEYd7YKVLFYC9zvmJUBEwhzTZJ5HRKoAXsaYT4GxQItsRc7OWni5431v4Dt7PoJSBadnJUrlwTHaaBCw3xgTD8SLSAPgJ+thJZKAXlhn+s7CgPdE5OyJ2JPZ9psiIv2Bj0XEB2so7Ok2fhSlCkQfc1VKKeWS3mJSSinlkiYIpZRSLmmCUEop5ZImCKWUUi5pglBKKeWSJgillFIuaYJQSinlkiYIpZRSLv0/1AE9lFxVOFsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "als = mdf[mdf['algo'] == 'ALS']\n",
    "sns.lineplot(x='version', y='nDCG', hue='data', data=als)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dccf855e",
   "metadata": {},
   "source": [
    "## Item-Item Results\n",
    "\n",
    "Now the item-item results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0d0deb38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:32.285686Z",
     "iopub.status.busy": "2021-06-18T16:47:32.285686Z",
     "iopub.status.idle": "2021-06-18T16:47:32.424780Z",
     "shell.execute_reply": "2021-06-18T16:47:32.424780Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='version', ylabel='GRMSE'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAa/ElEQVR4nO3dfZRU9Z3n8feHB0EjCi342JhG7NggD6LdaJ4UB7FlzaJREx/yJKOr7urEmInB5UQj49HVkXjCRDcEHwJGMp6jURYnGSGa4EN0heZJgmBE0dgYVx6aESaiEr/7R12waG9T1XTdrqb78zqHc+rW/d1b33sPpz/1u1X1vYoIzMzMmutW7gLMzKxjckCYmVkqB4SZmaVyQJiZWSoHhJmZpepR7gJKqX///lFVVVXuMszM9hqLFy/eEBED0tZ1qoCoqqqioaGh3GWYme01JL3R0jpfYjIzs1QOCDMzS+WAMDOzVJ3qMwgz67o+/PBDGhsb2bZtW7lL6ZB69+5NZWUlPXv2LHobB4SZdQqNjY306dOHqqoqJJW7nA4lIti4cSONjY0MGjSo6O18icnMOoVt27Zx0EEHORxSSOKggw5q9ezKAWFmnYbDoWV7cm58iQmY8thKXnrr3XKXYWZtcOWofdln/dZyl1EW+/bszuF99y35fj2DMDNrg2n/fAv33DWtxfW//c1jvPLy6nasqHQ8gwB++F+PLXcJZtZGq1atYvCA/dv9dSs+tQ/779+rxde+6ffz6P+pngz+Qm07V9Z2nkGYmbXSzTffzDHHHMNpp53Gyy+/DMDdd99NXV0dI0eO5Nxzz+Wvf/0rzz33HHPnzuXaa6/luOOO49VXX00d11E5IMzMWmHx4sU8+OCDLF26lEceeYRFixYBcM4557Bo0SKWL1/OkCFDuPfee/nc5z7HhAkTuP3221m2bBmDBw9OHddR+RKTmVkrPPPMM3z5y19mv/32A2DChAkA/PGPf+QHP/gBmzdvZuvWrdTX16duX+y4jsABYWbWSmlfGb344ouZM2cOI0eOZObMmSxYsCB122LHdQS+xGRm1gonn3wyjz76KO+99x5btmzhscceA2DLli0cdthhfPjhh8yePXvn+D59+rBly5adyy2N64gcEGZmrXD88cdz/vnnc9xxx3HuuefyxS9+EYCbbrqJE088kXHjxlFTU7Nz/AUXXMDtt9/OqFGjePXVV1sc1xEpIspdQ8nU1taGbxhk1jWtWrWKIUOGlLuMDi3tHElaHBGp38H1DMLMzFI5IMzMLJUDwszMUjkgzMwsVaYBIekMSS9LWiPpupT1/SQ9KulFSQslDUueHyjp95JWSVop6eos6zQzs0/KLCAkdQfuAsYDQ4ELJQ1tNmwysCwiRgDfBHa0RNwO/GNEDAFOAq5M2dbMzDKU5QxiNLAmIl6LiA+AB4Gzmo0ZCjwJEBGrgSpJh0TEXyJiSfL8FmAVcESGtZqZtYuZM2dy1VVXAfD0009z/PHH06NHDx5++OFdxs2aNYvq6mqqq6uZNWvWzufXrl3LiSeeSHV1Neeffz4ffPABADfeeCNTp04taa1ZBsQRwJt5y4188o/8cuAcAEmjgU8DlfkDJFUBo4AX0l5E0mWSGiQ1rF+/vjSVm5m1gyOPPJKZM2dy0UUX7fL8pk2bmDJlCi+88AILFy5kypQpNDU1ATBp0iSuueYaXnnlFfr165dps78sAyLt/nbNf5V3K9BP0jLgH4Cl5C4v5XYg7Q/8CvhORKTe8i0iZkREbUTUDhgwoCSFm5ntiddff52amhouvfRShg0bxte+9jWeeOIJPv/5z1NdXc3ChQt3GV9VVcWIESPo1m3XP8Xz5s1j3LhxVFRU0K9fP8aNG8fjjz9ORPC73/2O8847D4BvfetbzJkz5xN13H333YwfP5733nuvTceTZbO+RmBg3nIl8Fb+gOSP/kQA5bpfrU3+IaknuXCYHRGPZFinmXUyWdxGeOjhBxR1c7E1a9bw0EMPMWPGDOrq6vjlL3/Js88+y9y5c7nllls4++yzC+5j3bp1DBz48Z/PyspK1q1bx8aNG+nbty89evTY5fl8d955J/Pnz2fOnDn06tWrdQfZTJYBsQioljQIWAdcAOwyj5LUF/hr8hnFpcDTEfFuEhb3Aqsi4o4MazQzK6lBgwYxfPhwAI499ljGjh2LJIYPH87rr79e1D7SWiBJavH5HX7xi19QWVnJnDlz6Nmz554dQJ7MAiIitku6CpgHdAfui4iVkq5I1k8HhgD3S/ob8BJwSbL554FvACuSy08AkyPiN1nVa2adRzlvI5z/rr1bt247l7t168b27dtb2mwXlZWVu7QBb2xsZMyYMfTv35/Nmzezfft2evToQWNjI4cffvjOccOGDWPZsmU0NjYyaNCgNh9Lpr+DiIjfRMRnImJwRNycPDc9CQci4vmIqI6Imog4JyKakuefjQhFxIiIOC7553Awsy6hvr6e+fPn09TURFNTE/Pnz6e+vh5JnHrqqTu/8TRr1izOOuvjL4eOGjWKn/3sZ0yYMIG33nqrpd0Xzb+kNjMrk0WLFlFZWclDDz3E5ZdfzrHH5mY+FRUVXH/99dTV1VFXV8cNN9xARUUFALfddht33HEHRx99NBs3buSSSy7ZZZ9f+MIXmDp1KmeeeSYbNmxoU31u921mnYLbfRfmdt9mZlYSDggzM0vlgDAzs1QOCDMzS+WAMDOzVA4IMzNL5YAwM2tHxbb77ggcEGZmZdJSu++OwgFhZlYipWr3vWDBAk455RS++tWv8pnPfIbrrruO2bNnM3r0aIYPH86rr77aLseTZTdXM7Py+Pfr4O0Vpd3nocNh/K0Fh5Wi3TfA8uXLWbVqFRUVFRx11FFceumlLFy4kGnTpvGTn/yEH//4x207niJ4BmFmVkI72n1369Ztj9t9A9TV1XHYYYfRq1cvBg8ezOmnnw7Q6v20hWcQZtb5FPFOPyulaPddyv20hWcQZmaWygFhZlYmLbX77ijc7tvMOgW3+y7M7b7NzKwkHBBmZpbKAWFmZqkcEGZmlsoBYWZmqRwQZmaWygFhZtaOim33PWvWLKqrq6murmbWrFnlKNWtNszMymVHu++pU6fu8vymTZuYMmUKDQ0NSOKEE05gwoQJ9OvXr13r8wzCzKxEStXue968eYwbN46Kigr69evHuHHjePzxx3duM3nyZD772c9SW1vLkiVLqK+vZ/DgwUyfPr2kx+MZhJl1OrctvI3Vm1aXdJ81FTVMGj2p4LhStPtet24dAwcO3LlcWVnJunXrdi4PHDiQ559/nmuuuYaLL76YP/zhD2zbto1jjz2WK664Yo+OL40DwsyshHa0+wb2uN13WgskSTsfT5gwAci1/t66dSt9+vShT58+9O7dm82bN9O3b982Hwc4IMysEyrmnX5WStGmu7KykgULFuxcbmxsZMyYMZ94jfz9t/Y1iuHPIMzMOpj6+nrmz59PU1MTTU1NzJ8/n/r6+navwwFhZlYmLbX7rqio4Prrr6euro66ujpuuOEGKioq2r0+t/s2s07B7b4Lc7tvMzMrCQeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpco0ICSdIellSWskXZeyvp+kRyW9KGmhpGHFbmtmtjfKb/d9xx13MHToUEaMGMHYsWN54403do7rCO2+MwsISd2Bu4DxwFDgQklDmw2bDCyLiBHAN4FprdjWzGyvNmrUKBoaGnjxxRc577zz+P73vw983O77hRdeYOHChUyZMoWmpqZ2ry/LGcRoYE1EvBYRHwAPAmc1GzMUeBIgIlYDVZIOKXJbM7MOpbXtvk899VT2228/AE466SQaGxuBrtHu+wjgzbzlRuDEZmOWA+cAz0oaDXwaqCxyWwAkXQZcBrmbb5iZvX3LLby/qrTtvnsNqeHQyZMLjtvTdt/33nsv48ePB7pGu2+lPNe8r8etwDRJy4AVwFJge5Hb5p6MmAHMgFyrjT0t1sysFPak3fcDDzxAQ0MDTz31FNA12n03AgPzliuBt/IHRMS7wEQA5Y5+bfJvv0Lbmpm1pJh3+llpbbvvJ554gptvvpmnnnpq59iu0O57EVAtaZCkfYALgLn5AyT1TdYBXAo8nYRGwW3NzPZ2S5cu5fLLL2fu3LkcfPDBO5/vKO2+M5tBRMR2SVcB84DuwH0RsVLSFcn66cAQ4H5JfwNeAi7Z3bZZ1WpmVg7XXnstW7du5Stf+QqQ+xx17ty5u7T7BtzuuxTc7tus63K778Lc7tvMzErCAWFmZqkcEGbWaXSmS+altifnxgFhZp1C79692bhxo0MiRUSwceNGevfu3artsvwdhJlZu6msrKSxsZH169eXu5QOqXfv3lRWVrZqGweEmXUKPXv2ZNCgQeUuo1PxJSYzM0vlgDAzs1QOCDMzS+WAMDOzVA4IMzNL5YAwM7NUDggzM0vlgDAzs1QOCDMzS7XbgJB0wG7WHVn6cszMrKMoNINYsOOBpCebrZtT6mLMzKzjKBQQynvc/H53wszMOq1CAREtPE5bNjOzTqRQN9eDJX2X3Gxhx2OS5QGZVmZmZmVVKCDuBvqkPAa4J5OKzMysQ9htQETElPYqxMzMOpZCX3P9b5Kqk8eSdJ+k/5D0oqRR7VOimZmVQ6EPqa8GXk8eXwiMBI4Cvgv8S3ZlmZlZuRUKiO0R8WHy+EvA/RGxMSKeAD6VbWlmZlZOhQLiI0mHSeoNjAWeyFu3b3ZlmZlZuRX6FtMNQAPQHZgbESsBJJ0CvJZxbWZmVkaFvsX0b5I+DfSJiKa8VQ3A+ZlWZmZmZbXbgJB0Tt7jtCGPlLogMzPrGApdYnoYWJb8g137LwUOCDOzTqtQQJxL7lLSCOD/AP8aEWsyr8rMzMput99iiohHI+IC4BTgVeBHkp5NPqQ2M7NOrNg7ym0D/gN4l9zvH3pnVpGZmXUIhT6kPpXcL6hHk/sNxLSIaGiPwszMrLwKfQbxJPAi8CzQC/impG/uWBkR386wNjMzK6NCATGxXaowM7MOp9AP5Wa1tC75AZ2ZmXVSBT+klvRZSedJOjhZHiHpl+QuO5mZWSdV6H4QtwP3kfs9xK8l/RD4LfACUJ19eWZmVi6FPoM4ExgVEdsk9QPeAkZExCvF7FzSGcA0cs3+7omIW5utPxB4ADgyqWVqRPw8WXcNcCm5X2yvACZGxLaij8zMzNqk0CWm93b8UU6a9b3cinDoDtwFjAeGAhdKGtps2JXASxExEhhD7od4+0g6Avg2UBsRw8gFzAVFHpOZmZVAoRnEYElz85arkmUBERETdrPtaGBNRLwGIOlB4CzgpbwxAfRRrhPg/sAmYHtebftK+hDYj9zsxczM2kmhgDir2fKPyP1Rh10b96U5Angzb7kROLHZmDuBueT++PcBzo+Ij4B1kqYCfwbeA+ZHxPy0F5F0GXAZwJFHHlmgJDMzK1ahgOgLVEbEXQCSFgIDyIXEpALbpgVINFuuJ9cp9u+AwcBvJT1D7pLSWcAgYDPwkKSvR8QDn9hhxAxgBkBtbW3z/Rfn36+Dt1fs0aZmZmV36HAYf2vhca1U6DOI75N7h7/DPkAtuc8LriiwbSMwMG+5kk9eJpoIPBI5a4C1QA1wGrA2ItYn98R+BPhcgdczM7MSKjSD2Cci8i8TPRsRG4GNkj5VYNtFQLWkQcA6ch8yX9RszJ/J3ev6GUmHAMeQu5WpgJMk7UfuEtNYcnexy0YGyWtmtrcrFBD98hci4qq8xQG72zAitku6CphH7pLRfRGxUtIVyfrpwE3ATEkryIXCpIjYAGyQ9DCwhNyH1ktJLiOZmVn7UETLl+0lzQYWRMTdzZ6/HBgTERdmXF+r1NbWRkODm82amRVL0uKIqE1bV2gGcQ0wR9JF5N7NA5xArrPr2SWr0MzMOpxCzfreAT4n6e+AY5Onfx0Rv8u8MjMzK6tCMwgAkkBwKJiZdSHF3nLUzMy6GAeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpXJAmJlZKgeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpXJAmJlZKgeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpXJAmJlZKgeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpXJAmJlZKgeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpXJAmJlZKgeEmZmlckCYmVkqB4SZmaVyQJiZWSoHhJmZpco0ICSdIellSWskXZey/kBJj0laLmmlpIl56/pKeljSakmrJH02y1rNzGxXmQWEpO7AXcB4YChwoaShzYZdCbwUESOBMcCPJO2TrJsGPB4RNcBIYFVWtZqZ2SdlOYMYDayJiNci4gPgQeCsZmMC6CNJwP7AJmC7pAOAk4F7ASLig4jYnGGtZmbWTJYBcQTwZt5yY/JcvjuBIcBbwArg6oj4CDgKWA/8XNJSSfdI+lTai0i6TFKDpIb169eX/CDMzLqqLANCKc9Fs+V6YBlwOHAccGcye+gBHA/8NCJGAf8JfOIzDICImBERtRFRO2DAgBKVbmZmWQZEIzAwb7mS3Ewh30TgkchZA6wFapJtGyPihWTcw+QCw8zM2kmWAbEIqJY0KPng+QJgbrMxfwbGAkg6BDgGeC0i3gbelHRMMm4s8FKGtZqZWTM9stpxRGyXdBUwD+gO3BcRKyVdkayfDtwEzJS0gtwlqUkRsSHZxT8As5NweY3cbMPMzNqJIpp/LLD3qq2tjYaGhnKXYWa215C0OCJq09b5l9RmZpbKAWFmZqkcEGZmlsoBYWZmqRwQZmaWygFhZmapHBBmZpbKAWFmZqkcEGZmlsoBYWZmqRwQZmaWygFhZmapHBBmZpbKAWFmZqkcEGZmlsoBYWZmqRwQZmaWygFhZmapHBBmZpbKAWFmZqkcEGZmlsoBYWZmqRwQZmaWygFhZmapepS7gI7gtoW3sXrT6nKXYWa2R2oqapg0elLJ9+sZhJmZpfIMAjJJXjOzvZ1nEGZmlsoBYWZmqRwQZmaWygFhZmapHBBmZpbK32IC3r7lFt5f5d9BmNneqdeQGg6dPLnk+/UMwszMUnkGAZkkr5nZ3s4zCDMzS+WAMDOzVA4IMzNLlWlASDpD0suS1ki6LmX9gZIek7Rc0kpJE5ut7y5pqaR/y7JOMzP7pMwCQlJ34C5gPDAUuFDS0GbDrgReioiRwBjgR5L2yVt/NbAqqxrNzKxlWc4gRgNrIuK1iPgAeBA4q9mYAPpIErA/sAnYDiCpEjgTuCfDGs3MrAVZBsQRwJt5y43Jc/nuBIYAbwErgKsj4qNk3Y+B7wMfsRuSLpPUIKlh/fr1pajbzMzINiCU8lw0W64HlgGHA8cBd0o6QNKXgHciYnGhF4mIGRFRGxG1AwYMaGPJZma2Q5Y/lGsEBuYtV5KbKeSbCNwaEQGskbQWqAE+D0yQ9F+A3sABkh6IiK/v7gUXL168QdIbe1hvf2DDHm7bFfl8tY7PV+v4fLVOW87Xp1taodzf5tKT1AP4EzAWWAcsAi6KiJV5Y34K/L+IuFHSIcASYGREbMgbMwb4XkR8KZNCP36dhoiozfI1OhOfr9bx+Wodn6/Wyep8ZTaDiIjtkq4C5gHdgfsiYqWkK5L104GbgJmSVpC7JDUpPxzMzKx8Mu3FFBG/AX7T7LnpeY/fAk4vsI8FwIIMyjMzs93wL6k/NqPcBexlfL5ax+erdXy+WieT85XZZxBmZrZ38wzCzMxSOSDMzCxVlwqIIpoH9pP0qKQXJS2UNKwcdZZTEeeoRtLzkt6X9L3WbNsZtfF83SfpHUl/bL+Ky29Pz5mkgZJ+L2lV0tzz6vatfO8hqVbSv7R5P13lM4ikeeCfgHHkfsS3CLgwIl7KG3M7sDUipkiqAe6KiLFlKbgMijxHB5P7Yc3ZQFNETC12286mLecrWXcysBW4PyK6xJuRNv4fOww4LCKWSOoDLAbO7sz/x8qtK80gimkeOBR4EiAiVgNVyQ/4uoqC5ygi3omIRcCHrd22E2rL+SIinibXoLIr2eNzFhF/iYglyeMt5Do9N+/v1mlIqpK0WtI9kv4oabak0yT9QdIrkkYn/55LbovwnKRjkm3H7LhNgqQbk9nqAkmvSfp2sTV0pYAopnngcuAcAEmjyb2LqWyX6jqGYs5RFtvurbriMbdVSc6ZpCpgFPBCacrqsI4GpgEjyLUhugj4AvA9YDKwGjg5IkYBNwC3tLCfGnK970YDP5TUs5gXz/SHch1MMc0DbwWmSVpGrrvsUpL2411EMecoi233Vl3xmNuqzedM0v7Ar4DvRMS7Jamq41obESsAJK0EnoyISLpPVAEHArMkVZM7jy394f91RLwPvC/pHeAQcuG8W10pIAo2D0z+s00ESO5RsTb511UU02Axi233Vl3xmNuqTecseef7K2B2RDxS4to6ovfzHn+Ut/wRub/fNwG/j4gvJ7OqBUXs528U+be/K11iWgRUSxqU3LXuAmBu/gBJffXxHe0uBZ7uAu9Q8hU8Rxltu7fqisfcVnt8zpI3bfcCqyLijgxr3JscSK4ZKsDFpd55l5lBFNk8cAhwv6S/AS8Bl5St4DIo5hxJOhRoAA4APpL0HWBoRLybtm1ZDqSdlOB8/Su5W+32l9QI/DAi7i3HsbSXtpwzctfhvwGsSC4DA0xOer51Vf9M7hLTd4HflXrnXeZrrmZm1jpd6RKTmZm1ggPCzMxSOSDMzCyVA8LMzFI5IMzMLJUDwqydSfonSaeVuw6zQvw1V7MMJD/qUkR8VO5azPaUZxBmuyHpNkn/I2/5Rkn/KOlaSYuUu3fIlGRdVXKvgv8NLAEGSpqZdOJcIemaZNxMSeclj8cmnThXJB03eyXPvy5piqQlybqa9j966+ocEGa79yBwft7yV4H1QDW5zpjHASck93YAOIbc/R1GAf2BIyJiWEQMB36ev2NJvYGZwPnJ+h7Af88bsiEijgd+Sq57p1m7ckCY7UZELAUOlnS4pJFAE7mWD6eT6/a7hFwr5epkkzci4v8mj18DjpL0E0lnAM37eh1Drlvnn5LlWcDJeet3NKNbTK5zp1m76jK9mMza4GHgPOBQcjOKKuB/RcTP8gcl3TT/c8dyRDQloVIPXElu9vH3+ZsUeN0dHTiL7r5pVkqeQZgV9iC5rqPnkQuLecDfJ/clQNIRyW0ydyGpP9AtIn4FXA8c32zIjrsWHp0sfwN4KptDMGs9vysxKyDpNtoHWBcRfwH+ImkI8Hzuy0psBb5O7p1+viOAn0va8Ubsfzbb7zZJE4GHJPUg1wp7eoaHYtYq/pqrmZml8iUmMzNL5YAwM7NUDggzM0vlgDAzs1QOCDMzS+WAMDOzVA4IMzNL9f8BKAmA+gC8BxEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ii_exp = mdf[mdf['algo'] == 'II']\n",
    "sns.lineplot(x='version', y='GRMSE', hue='data', data=ii_exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6b5aff04",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:32.442779Z",
     "iopub.status.busy": "2021-06-18T16:47:32.442779Z",
     "iopub.status.idle": "2021-06-18T16:47:32.581778Z",
     "shell.execute_reply": "2021-06-18T16:47:32.580778Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='version', ylabel='nDCG'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAavklEQVR4nO3de5RU5Z3u8e/DRRkzJNBKItowTUxHLqKC3WBiYjRKgFxAo4maTBKdeJCJrJk4GS/jihomSyceXS4v8WgwZsDEDCsmkdVmHGGMQXNR6Ua8ITppEU8ayRGxMRBFIfmdP2pDivalq/qyu/ryfNaqZe+933fXr7Yun3rr8itFBGZmZm0NqnQBZmbWOzkgzMwsyQFhZmZJDggzM0tyQJiZWdKQShfQnQ466KCoqampdBlmZn3G6tWrX4mIUalj/SogampqaGpqqnQZZmZ9hqQX93XMLzGZmVmSA8LMzJIcEGZmluSAMDOzJAeEmZklOSDMzCzJAWFmZkn96nsQnbXwnrU889IfKl2GmVmnTDzknVzxqUndfl6vIMzMLMkrCMglec3M+jqvIMzMLMkBYWZmSQ4IMzNLckCYmVmSA8LMzJIcEGZmluSAMDOzJAeEmZklOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWVKuASFplqTnJDVLuiRxXJJuzI4/KWlq0bENkp6S9LikpjzrNDOztxuS14klDQZuBmYALUCjpIaIeKZo2GygNrtNB27J/rnbiRHxSl41mpnZvuW5gpgGNEfE+oh4C1gKzG0zZi5wRxQ8AoyQNDrHmszMrEx5BsShwO+KtluyfeWOCWCFpNWS5u3rTiTNk9QkqWnz5s3dULaZmUG+AaHEvujAmOMiYiqFl6HOl3R86k4iYlFE1EVE3ahRozpfrZmZ7SXPgGgBxhRtVwMvlTsmInb/82XgbgovWZmZWQ/JMyAagVpJ4yTtB5wJNLQZ0wB8Mfs007HAaxGxSdI7JA0HkPQO4GPA0znWamZmbeT2KaaI2CVpAbAcGAx8LyLWSpqfHb8VuBf4ONAMvA6ck01/D3C3pN01/jAi7surVjMzeztFtH1boO+qq6uLpiZ/ZcLMrFySVkdEXeqYv0ltZmZJDggzM0tyQJiZWZIDwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLMkBYWZmSQ4IMzNLckCYmVmSA8LMzJIcEGZmluSAMDOzJAeEmZklOSDMzCwpt1+U61P+6xL4/VOVrsLMrHMOngyzv9Xtp/UKwszMkryCgFyS18ysr/MKwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLMkBYWZmSQ4IMzNLckCYmVlSrgEhaZak5yQ1S7okcVySbsyOPylpapvjgyWtkfSzPOs0M7O3yy0gJA0GbgZmAxOBsyRNbDNsNlCb3eYBt7Q5/o/AurxqNDOzfcuz1cY0oDki1gNIWgrMBZ4pGjMXuCMiAnhE0ghJoyNik6Rq4BPAlcA/5VinmfUDO3fupKWlhR07dlS6lF5p2LBhVFdXM3To0LLn5BkQhwK/K9puAaaXMeZQYBNwPXARMLy9O5E0j8Lqg7Fjx3apYDPru1paWhg+fDg1NTVIqnQ5vUpEsGXLFlpaWhg3blzZ8/J8DyL1byjKGSPpk8DLEbG61J1ExKKIqIuIulGjRnWmTjPrB3bs2MGBBx7ocEiQxIEHHtjh1VWeAdECjCnargZeKnPMccAcSRuApcBHJf0gv1LNrD9wOOxbZ65NngHRCNRKGidpP+BMoKHNmAbgi9mnmY4FXouITRHxLxFRHRE12bwHIuJvc6zVzMzayC0gImIXsABYTuGTSD+KiLWS5kuanw27F1gPNAO3AV/Jqx4zszx84xvf4Nprr93n8WXLlvHMM8/s83hvlusPBkXEvRRCoHjfrUV/B3B+iXOsBFbmUJ6ZWe6WLVvGJz/5SSZObPsp/97P36Q2M+ugK6+8ksMPP5yTTz6Z5557DoDbbruN+vp6jjrqKE477TRef/11fvOb39DQ0MCFF17I0UcfzfPPP58c11s5IMzMOmD16tUsXbqUNWvW8NOf/pTGxkYAPv3pT9PY2MgTTzzBhAkTuP322/ngBz/InDlzuOaaa3j88cc57LDDkuN6K/8mtZlZB/zyl7/k1FNP5YADDgBgzpw5ADz99NN8/etfZ+vWrWzfvp2ZM2cm55c7rjdwQJiZdVDqI6Nnn302y5Yt46ijjmLx4sWsXLkyObfccb2BX2IyM+uA448/nrvvvps33niDbdu2cc899wCwbds2Ro8ezc6dO7nzzjv3jB8+fDjbtm3bs72vcb2RA8LMrAOmTp3KGWecwdFHH81pp53Ghz/8YQC++c1vMn36dGbMmMH48eP3jD/zzDO55pprmDJlCs8///w+x/VGKnzStH+oq6uLpqamSpdhZhWwbt06JkyYUOkyerXUNZK0OiLqUuO9gjAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwsysBy1evJgFCxYA8NBDDzF16lSGDBnCj3/8473GLVmyhNraWmpra1myZMme/S+88ALTp0+ntraWM844g7feegso3Xa8MxwQZmYVMnbsWBYvXsznPve5vfa/+uqrLFy4kEcffZRVq1axcOFCWltbAbj44ou54IIL+O1vf8vIkSNzbfbngDAz6yYbNmxg/PjxnHvuuRxxxBF8/vOf5/777+e4446jtraWVatW7TW+pqaGI488kkGD9v5f8fLly5kxYwZVVVWMHDmSGTNmcN999xERPPDAA5x++ukAfOlLX2LZsmVvq+O2225j9uzZvPHGG116PG7WZ2b9zsJ71vLMS3/o1nNOPOSdXPGpSSXHNTc3c9ddd7Fo0SLq6+v54Q9/yK9+9SsaGhq46qqrOOWUU0qeY+PGjYwZM2bPdnV1NRs3bmTLli2MGDGCIUOG7LW/2Le//W1WrFjBsmXL2H///Tv2INtwQJiZdaNx48YxefJkACZNmsRJJ52EJCZPnsyGDRvKOkeqBZKkfe7f7fvf/z7V1dUsW7aMoUOHdu4BFHFAmFm/U84z/bwUP2sfNGjQnu1Bgwaxa9euss5RXV29VxvwlpYWTjjhBA466CC2bt3Krl27GDJkCC0tLRxyyCF7xh1xxBE8/vjjtLS0MG7cuC4/Fr8HYWbWy8ycOZMVK1bQ2tpKa2srK1asYObMmUjixBNP3POJpyVLljB37tw986ZMmcJ3vvMd5syZw0svvdTlOhwQZmYV0tjYSHV1NXfddRfnnXcekyYVVj5VVVVcdtll1NfXU19fz+WXX05VVRUAV199Nddddx3ve9/72LJlC1/+8pf3OueHPvQhrr32Wj7xiU/wyiuvdKm+dtt9S5oLVEfEzdn2o8Co7PBFEfHjfU6uALf7Nhu43O67tO5u930R0FC0vT9QD5wA/H3nyzQzs96u1JvU+0XE74q2fxURW4Atkt6RY11mZlZhpVYQI4s3ImJB0eYozMys3yoVEI9K+l9td0o6D1iVGG9mZv1EqZeYLgCWSfoc8Fi27xgK70WckmNdZmZWYe2uICLi5Yj4IPBNYEN2+9eI+EBE/L9SJ5c0S9JzkpolXZI4Lkk3ZseflDQ12z9M0ipJT0haK2lhZx6cmZl1XrsBIale0uyIeCAibspuD0j6lKRjSswdDNwMzAYmAmdJmthm2GygNrvNA27J9r8JfDQijgKOBmZJOrajD87MrLcpt913b1DqPYhrgHWJ/euyY+2ZBjRHxPqIeAtYCsxtM2YucEcUPAKMkDQ6296ejRma3fb9hQ0zsz5oX+2+e4tSAXFgRGxouzMimoEDS8w9FCj+iGxLtq+sMZIGS3oceBn474h4NHUnkuZJapLUtHnz5hIlmZnlp7vafa9cuZKPfOQjfPazn+X9738/l1xyCXfeeSfTpk1j8uTJPP/88z3yeEq9Sf1X7Rwr9T0IJfa1XQXsc0xE/Ak4WtII4G5JR0TE028bHLEIWASFb1KXqMnMBoL/ugR+/1T3nvPgyTD7WyWHdUe7b4AnnniCdevWUVVVxXvf+17OPfdcVq1axQ033MBNN93E9ddf37XHU4ZSK4j7JV2p4n6yQPam8QMl5rYAY4q2q4G23aNKjomIrcBKYFaJ+zMzq7jd7b4HDRrU6XbfAPX19YwePZr999+fww47jI997GMAHT5PV5RaQXwN+C7QnL3cA4U3jRuBc0vMbQRqJY0DNgJnAm1faGsAFkhaCkwHXouITZJGATsjYqukvwJOBq4u7yGZ2YBXxjP9vHRHu+/uPE9XtBsQEfFHCp8+ei+wu8H62ohYX+rEEbFL0gJgOTAY+F5ErJU0Pzt+K3Av8HGgGXgdOCebPhpYkn0SahDwo4j4WYcfnZmZdVrJHwySNIRCOIzPdoWk/xsRJSMsIu6lEALF+24t+juA8xPzngSmlDq/mVlf1tjYyKmnnkprayv33HMPV1xxBWvXrq10WXuUavd9CPALYBOwhsKbylOAg4ETI6Lrv0jRjdzu22zgcrvv0jra7rvUCuIq4JaIuL7NCf8B+DfgS50v1czMerNSAXFsRJzddmdE3CjpuXxKMjOz3qDUx1zfaOfY691ZiJmZ9S6lVhDvkvTpxH4B78yhHjMz6yVKBcSDwKeKtne/o63smJmZ9VOlvgdxDhTabwOnATVFc9zWwsysHyv1HsRuyyisJHYC27PbH3Oqycys3yq33feSJUuora2ltraWJUuWVKLU0l+Uy1RHhHshmZl1o93tvq+99tq99r/66qssXLiQpqYmJHHMMccwZ84cRo4c2aP1lbuC+I2kyblWYmbWx3VXu+/ly5czY8YMqqqqGDlyJDNmzOC+++7bM+fSSy/lAx/4AHV1dTz22GPMnDmTww47jFtvvZXuVO4K4kPA2ZJeoPBrb6LQKePIbq3GzKwbXL3qap599dluPef4qvFcPO3ikuO6o933xo0bGTPmL42uq6ur2bhx457tMWPG8PDDD3PBBRdw9tln8+tf/5odO3YwadIk5s+f36nHl1JuQMzutns0M+vHdrf7Bjrd7jvVAqn4VxfmzJkDFFp/b9++neHDhzN8+HCGDRvG1q1bGTFiRJcfB5QZEBHxYrfcm5lZDyjnmX5euqNNd3V1NStXrtyz3dLSwgknnPC2+yg+f0fvoxzlvgdhZmY9ZObMmaxYsYLW1lZaW1tZsWIFM2fO7PE6HBBmZhXS2NhIdXU1d911F+eddx6TJhV+dqeqqorLLruM+vp66uvrufzyy6mqqurx+tpt993XuN232cDldt+ldbTdt1cQZmaW5IAwM7MkB4SZmSU5IMzMLMkBYWZmSQ4IMzNLckCYmfWg4nbf1113HRMnTuTII4/kpJNO4sUX/9K0oje0+3ZAmJlVyJQpU2hqauLJJ5/k9NNP56KLLgL+0u770UcfZdWqVSxcuJDW1tYer88BYWbWTTra7vvEE0/kgAMOAODYY4+lpaUF6Hvtvs3M+ozfX3UVb67r3nbf+08Yz8GXXlpyXGfbfd9+++3Mnl1onN3X2n2bmVkZOtPu+wc/+AFNTU08+OCDQB9r991ZkmYBNwCDge9GxLfaHFd2/OPA68DZEfGYpDHAHcDBwJ+BRRFxQ561mln/Uc4z/bx0tN33/fffz5VXXsmDDz64Z2y/b/ctaTBwM4UfG5oInCVpYpths4Ha7DYPuCXbvwv4WkRMAI4Fzk/MNTPr09asWcN5551HQ0MD7373u/fs7y3tvvNcQUwDmiNiPYCkpcBc4JmiMXOBO6KwnnpE0ghJoyNiE7AJICK2SVoHHNpmrplZn3bhhReyfft2PvOZzwAwduxYGhoa9mr3DfS/dt+STgdmRcS52fYXgOkRsaBozM+Ab0XEr7LtnwMXR0RT0Zga4CHgiIj4Q+J+5lFYfTB27Nhjij9HbGYDh9t9l9ab2n0rsa9tGrU7RtJfAz8BvpoKB4CIWBQRdRFRN2rUqE4Xa2Zme8szIFqAMUXb1cBL5Y6RNJRCONwZET/NsU4zM0vIMyAagVpJ4yTtB5wJNLQZ0wB8UQXHAq9FxKbs0023A+si4rocazSzfqQ//UJmd+vMtcktICJiF7AAWA6sA34UEWslzZe0+5sc9wLrgWbgNuAr2f7jgC8AH5X0eHb7eF61mlnfN2zYMLZs2eKQSIgItmzZwrBhwzo0z79JbWb9ws6dO2lpaWHHjh2VLqVXGjZsGNXV1QwdOnSv/e29Se1vUptZvzB06FDGjRtX6TL6FTfrMzOzJAeEmZklOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkvx7EMDVq67m2VefrXQZZmadMr5qPBdPu7jbz+sVhJmZJXkFAbkkr5lZX+cVhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkhwQZmaW5IAwM7OkXANC0ixJz0lqlnRJ4rgk3Zgdf1LS1KJj35P0sqSn86zRzMzScgsISYOBm4HZwETgLEkT2wybDdRmt3nALUXHFgOz8qrPzMzal+cKYhrQHBHrI+ItYCkwt82YucAdUfAIMELSaICIeAh4Ncf6zMysHXkGxKHA74q2W7J9HR3TLknzJDVJatq8eXOnCjUzs7fLMyCU2BedGNOuiFgUEXURUTdq1KiOTDUzs3bkGRAtwJii7WrgpU6MMTOzCsgzIBqBWknjJO0HnAk0tBnTAHwx+zTTscBrEbEpx5rMzKxMuQVEROwCFgDLgXXAjyJiraT5kuZnw+4F1gPNwG3AV3bPl/QfwMPA4ZJaJH05r1rNzOztFNGhl/x7tbq6umhqaqp0GWZmfYak1RFRlzrmb1KbmVmSA8LMzJIcEGZmluSAMDOzJAeEmZklOSDMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzsyQHhJmZJTkgzMwsyQFhZmZJDggzM0tyQJiZWZIDwszMkhwQZmaW5IAwM7MkB4SZmSU5IMzMLGlIpQvoDX5/1VW8ue7ZSpdhZtYp+08Yz8GXXtrt5/UKwszMkryCgFyS18ysr/MKwszMkhwQZmaWlGtASJol6TlJzZIuSRyXpBuz409KmlruXDMzy1duASFpMHAzMBuYCJwlaWKbYbOB2uw2D7ilA3PNzCxHea4gpgHNEbE+It4ClgJz24yZC9wRBY8AIySNLnOumZnlKM+AOBT4XdF2S7avnDHlzAVA0jxJTZKaNm/e3OWizcysIM+AUGJflDmmnLmFnRGLIqIuIupGjRrVwRLNzGxf8vweRAswpmi7GnipzDH7lTHXzMxylGdANAK1ksYBG4Ezgc+1GdMALJC0FJgOvBYRmyRtLmPu26xevfoVSS92st6DgFc6OXcg8vXqGF+vjvH16piuXK+/2deB3AIiInZJWgAsBwYD34uItZLmZ8dvBe4FPg40A68D57Q3t4z77PRrTJKaIqKus/MHGl+vjvH16hhfr47J63rl2mojIu6lEALF+24t+juA88uda2ZmPcffpDYzsyQHxF8sqnQBfYyvV8f4enWMr1fH5HK9VHiVx8zMbG9eQZiZWZIDwszMkgZUQJTRXXakpLuzzrKrJB1RiTorqYxrNF7Sw5LelPTPHZnbH3Xxen1P0suSnu65iiuvs9dM0hhJv5C0TtJaSf/Ys5X3HZLqJN3Y5fMMlPcgsg6x/wPMoPAN7kbgrIh4pmjMNcD2iFgoaTxwc0ScVJGCK6DMa/RuCl+sOQVojYhry53b33TlemXHjge2U2hYOSCejHTxv7HRwOiIeEzScGA1cEp//m+s0gbSCqKcDrETgZ8DRMSzQI2k9/RsmRVV8hpFxMsR0Qjs7Ojcfqgr14uIeAh4tUcq7T06fc0iYlNEPJb9vQ1Yxz6aePYHkmokPSvpu5KelnSnpJMl/VrSbyVNy26/kbQm++fh2dwTJP0s+/sb2Wp1paT1kv6h3BoGUkCU0yH2CeDTAJKmUXgWU90j1fUOZXfR7ea5fdVAfMxd1S3XTFINMAV4tHvK6rXeB9wAHAmMp9By6EPAPwOXAs8Cx0fEFOBy4Kp9nGc8MJNCQF8haWg5d57rN6l7mXI6xH4LuEHS48BTwBpgV8519SZld9Ht5rl91UB8zF3V5Wsm6a+BnwBfjYg/dEtVvdcLEfEUgKS1wM8jIiQ9BdQA7wKWSKqlcB339T/+/4yIN4E3Jb0MvIdCOLdrIAVEye6y2X9s50Dh51CBF7LbQFFOB9485vZVA/Exd1WXrln2zPcnwJ0R8dNurq03erPo7z8Xbf+Zwv+/vwn8IiJOzVZVK8s4z58o8//9A+klpj3dZSXtR6FDbEPxAEkjsmMA5wIPDYBnKMVKXqOc5vZVA/Exd1Wnr1n2pO12YF1EXJdjjX3Juyh0vAY4u7tPPmBWEGV2l50A3CHpT8AzwJcrVnAFlHONJB0MNAHvBP4s6avAxIj4Q2puRR5ID+mG6/UfwAnAQZJagCsi4vZKPJae0pVrRuF1+C8AT2UvAwNcmjX2HKj+N4WXmP4JeKC7Tz5gPuZqZmYdM5BeYjIzsw5wQJiZWZIDwszMkhwQZmaW5IAwM7MkB4RZD5P0r5JOrnQdZqX4Y65mOci+1KWI+HOlazHrLK8gzNoh6WpJXyna/oakr0m6UFKjCr8dsjA7VpP9VsH/AR4DxkhanHXifErSBdm4xZJOz/4+KevE+VTWcXP/bP8GSQslPZYdG9/zj94GOgeEWfuWAmcUbX8W2AzUUuiMeTRwTPbbDgCHU/h9hynAQcChEXFEREwG/r34xJKGAYuBM7LjQ4C/LxrySkRMBW6h0L3TrEc5IMzaERFrgHdLOkTSUUArhZYPH6PQ7fcxCq2Ua7MpL0bEI9nf64H3SrpJ0iygbV+vwyl06/yfbHsJcHzR8d3N6FZT6Nxp1qMGTC8msy74MXA6cDCFFUUN8G8R8Z3iQVk3zT/u3o6I1ixUZgLnU1h9/F3xlBL3u7sDZ9ndN826k1cQZqUtpdB19HQKYbEc+LvsdwmQdGj2M5l7kXQQMCgifgJcBkxtM2T3rxa+L9v+AvBgPg/BrOP8rMSshKzb6HBgY0RsAjZJmgA8XPiwEtuBv6XwTL/YocC/S9r9ROxf2px3h6RzgLskDaHQCvvWHB+KWYf4Y65mZpbkl5jMzCzJAWFmZkkOCDMzS3JAmJlZkgPCzMySHBBmZpbkgDAzs6T/D34/h9cSgbTQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ii_exp = mdf[mdf['algo'] == 'II']\n",
    "sns.lineplot(x='version', y='nDCG', hue='data', data=ii_exp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d378f14d",
   "metadata": {},
   "source": [
    "## User-User Results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fb4bee55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:32.618778Z",
     "iopub.status.busy": "2021-06-18T16:47:32.612824Z",
     "iopub.status.idle": "2021-06-18T16:47:32.740779Z",
     "shell.execute_reply": "2021-06-18T16:47:32.741778Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='version', ylabel='GRMSE'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAe3ElEQVR4nO3dfZQV1Znv8e+PBsQXAo10jNJoE+wEEF/QBkeTqAnB1okXX+JSzKtGYrhXHGNm7khc8YVxJepIsmKi9xISGcBrwrq+wMXECImGEHON0EgjIjoimNiQiYgYZK6IDc/94xTkcKjmdNOn+jTdv89aZ/XZtfeueqpgnefsqjq7FBGYmZkV6lHuAMzMrHNygjAzs1ROEGZmlsoJwszMUjlBmJlZqp7lDqCUBg4cGDU1NeUOw8zsoLF8+fI3I6Iqra5LJYiamhoaGhrKHYaZ2UFD0h9bqvMpJjMzS+UEYWZmqTJNEJLOk/SypLWSpqTUV0qaJ+l5SUsljcyre03SKkmNknzeyMysg2V2DUJSBXAfMA5oApZJWhARL+Y1uwlojIiLJQ1L2o/Nq/9kRLyZVYxm1nW8//77NDU1sX379nKH0in16dOH6upqevXq1eo+WV6kHgOsjYh1AJLmAhcC+QliBHAHQES8JKlG0lER8ZcM4zKzLqipqYm+fftSU1ODpHKH06lEBJs3b6apqYkhQ4a0ul+Wp5gGAa/nlZuSZflWApcASBoDHAdUJ3UBLJK0XNI1LW1E0jWSGiQ1bNq0qWTBm9nBZfv27Rx55JFODikkceSRR7Z5dJVlgkj7VyqcOvZOoFJSI3AdsAJoTuo+FhGnAucD10o6K20jETEjIuoioq6qKvVWXjPrJpwcWnYgxybLU0xNwOC8cjWwMb9BRGwFrgJQLvr1yYuI2Jj8fUPSPHKnrJZkEejUx1bz4satWazazDrItaMOpfembeUOoywO7VXBMf0PLfl6sxxBLANqJQ2R1BuYACzIbyCpf1IHMBFYEhFbJR0uqW/S5nDgXOCFDGM1Mzsg9/zrd/jJffe0WP+rxx/jlZdf6sCISiezEURENEuaDCwEKoCZEbFa0qSkfjowHJgjaSe5i9dXJ92PAuYlQ6KewE8j4omsYr31v5yQ1arNrIOsWbOGoVVHdPh2BxzemyOOOKTFbd/+m4UMPLwXQz9e18GRtZ+60hPl6urqwlNtmHVPa9asYfjw4R2yrW9/+9vMmTOHwYMHU1VVxWmnnUa/fv2YMWMGO3bs4Pjjj+eBBx6gsbGRCy64gH79+tGvXz8eeeQRnnrqqX3aHXbYYR0Sd9oxkrQ8IlKzl39JbWbWBsuXL2fu3LmsWLGCRx99lGXLlgFwySWXsGzZMlauXMnw4cO5//77OfPMMxk/fjx33303jY2NDB06NLVdZ9WlJuszM8va7373Oy6++OI93/rHjx8PwAsvvMC3vvUt3n77bbZt20Z9fX1q/9a26wycIMzM2ijtltErr7yS+fPnc/LJJzNr1iwWL16c2re17ToDn2IyM2uDs846i3nz5vHuu+/yzjvv8NhjjwHwzjvvcPTRR/P+++/z4IMP7mnft29f3nnnnT3lltp1Rk4QZmZtcOqpp3L55Zdzyimn8NnPfpZPfOITANx+++2cfvrpjBs3jmHDhu1pP2HCBO6++25GjRrFq6++2mK7zsh3MZlZl9CRdzEdrHwXk5mZlYQThJmZpXKCMDOzVE4QZmaWygnCzMxSOUGYmVkqJwgzsw40a9YsJk+eDMCSJUs49dRT6dmzJw8//PBe7WbPnk1tbS21tbXMnj17z/L169dz+umnU1tby+WXX86OHTsAuO2225g2bVpJY3WCMDMrk2OPPZZZs2bxuc99bq/lb731FlOnTuXZZ59l6dKlTJ06lS1btgBw4403csMNN/DKK69QWVmZ6WR/ThBmZiXy2muvMWzYMCZOnMjIkSP5/Oc/z69//Ws+9rGPUVtby9KlS/dqX1NTw0knnUSPHnt/FC9cuJBx48YxYMAAKisrGTduHE888QQRwVNPPcWll14KwJe//GXmz5+/Txw//vGPOf/883n33XfbtT+erM/MupwsHiM84pgPtOrhYmvXruWhhx5ixowZjB49mp/+9Kc8/fTTLFiwgO985ztcdNFFRdexYcMGBg/+2xObq6ur2bBhA5s3b6Z///707Nlzr+X57r33XhYtWsT8+fM55JBD2raTBZwgzMxKaMiQIZx44okAnHDCCYwdOxZJnHjiibz22mutWkfaFEiSWly+2wMPPEB1dTXz58+nV69eB7YDeZwgzKzLKedjhPO/tffo0WNPuUePHjQ3N7dqHdXV1XtNA97U1MQ555zDwIEDefvtt2lubqZnz540NTVxzDHH7Gk3cuRIGhsbaWpqYsiQIe3el0yvQUg6T9LLktZKmpJSXylpnqTnJS2VNLKgvkLSCkk/zzJOM7POpL6+nkWLFrFlyxa2bNnCokWLqK+vRxKf/OQn99zxNHv2bC688MI9/UaNGsWPfvQjxo8fz8aNG9sdR2YJQlIFcB9wPjACuELSiIJmNwGNEXES8CXgnoL664E1WcVoZlZOy5Yto7q6moceeoivfe1rnHBCbuQzYMAAbr75ZkaPHs3o0aO55ZZbGDBgAAB33XUX3/ve9zj++OPZvHkzV1999V7r/PjHP860adP4zGc+w5tvvtmu+DKb7lvSGcBtEVGflL8JEBF35LX5BXBHRDydlF8FzoyIv0iqBmYD3wa+EREXFNump/s267483XdxnWm670HA63nlpmRZvpXAJQCSxgDHAdVJ3feBfwZ27W8jkq6R1CCpYdOmTSUI28zMINsEse9DW6FwuHInUCmpEbgOWAE0S7oAeCMilhfbSETMiIi6iKirqqpqb8xmZpbI8i6mJmBwXrka2OuqSURsBa4CUO5erfXJawIwXtLfA32AD0j6XxHxhQzjNTOzPFmOIJYBtZKGSOpN7kN/QX4DSf2TOoCJwJKI2BoR34yI6oioSfo95eRgZtaxMhtBRESzpMnAQqACmBkRqyVNSuqnA8OBOZJ2Ai8CV7e4QjMz61CZ/lAuIh4HHi9YNj3v/TNAbZF1LAYWZxCemZnthyfrMzPrQK2d7rszcIIwMyuTlqb77iycIMzMSqRU030vXryYs88+m8suu4yPfOQjTJkyhQcffJAxY8Zw4okn8uqrr3bI/niyPjPren45Bf5jVWnX+aET4fw7izYrxXTfACtXrmTNmjUMGDCAD3/4w0ycOJGlS5dyzz338MMf/pDvf//77dufVvAIwsyshHZP992jR48Dnu4bYPTo0Rx99NEccsghDB06lHPPPRegzetpD48gzKzracU3/ayUYrrvUq6nPTyCMDOzVE4QZmZl0tJ0351FZtN9l4On+zbrvjzdd3GdabpvMzM7iDlBmJlZKicIMzNL5QRhZmapnCDMzCyVE4SZmaVygjAz60Ctne579uzZ1NbWUltby+zZs8sRqqfaMDMrl93TfU+bNm2v5W+99RZTp06loaEBSZx22mmMHz+eysrKDo3PIwgzsxIp1XTfCxcuZNy4cQwYMIDKykrGjRvHE088safPTTfdxBlnnEFdXR3PPfcc9fX1DB06lOnTp1NKmY4gJJ0H3EPumdQ/iYg7C+orgZnAUGA78JWIeEFSH2AJcEgS48MRcWuWsZpZ13HX0rt46a2XSrrOYQOGceOYG4u2K8V03xs2bGDw4MF7ytXV1WzYsGFPefDgwTzzzDPccMMNXHnllfz+979n+/btnHDCCUyaNOmA9i9NZglCUgVwHzAOaAKWSVoQES/mNbsJaIyIiyUNS9qPBd4DPhUR2yT1Ap6W9MuI+ENW8ZqZlcLu6b6BA57uO20KJEl73o8fPx7ITf29bds2+vbtS9++fenTpw9vv/02/fv3b/d+QLYjiDHA2ohYByBpLnAhkJ8gRgB3AETES5JqJB0VEX8BtiVteiWvrjNplJllqjXf9LNSimm6q6urWbx48Z5yU1MT55xzzj7byF9/W7fRGllegxgEvJ5XbkqW5VsJXAIgaQxwHFCdlCskNQJvAL+KiGfTNiLpGkkNkho2bdpU2j0wMyuD+vp6Fi1axJYtW9iyZQuLFi2ivr6+w+PIMkEoZVnhKOBOoDJJBNcBK4BmgIjYGRGnkEsYYySNTNtIRMyIiLqIqKuqqipV7GZmmWtpuu8BAwZw8803M3r0aEaPHs0tt9zCgAEDOjy+zKb7lnQGcFtE1CflbwJExB0ttBewHjgpIrYW1N0K/GdETEvru5un+zbrvjzdd3GdabrvZUCtpCGSegMTgAUFgfVP6gAmAksiYqukKkn9kzaHAp8GSntLgpmZ7VdmF6kjolnSZGAhudtcZ0bEakmTkvrpwHBgjqSd5C5eX510PxqYndwJ1QP43xHx86xiNTOzfWX6O4iIeBx4vGDZ9Lz3zwC1Kf2eB0ZlGZuZdT0RsdftoPY3B3I5wb+kNrMuoU+fPmzevPmAPgi7uohg8+bN9OnTp039PBeTmXUJ1dXVNDU14dvd0/Xp04fq6uo29XGCMLMuoVevXgwZMqTcYXQpPsVkZmapnCDMzCyVE4SZmaVygjAzs1ROEGZmlsoJwszMUjlBmJlZKicIMzNL5QRhZmapnCDMzCyVE4SZmaVygjAzs1ROEGZmlsoJwszMUjlBmJlZKicIMzNLlWmCkHSepJclrZU0JaW+UtI8Sc9LWippZLJ8sKTfSFojabWk67OM08zM9pVZgpBUAdwHnA+MAK6QNKKg2U1AY0ScBHwJuCdZ3gz8Y0QMB/4OuDalr5mZZSjLEcQYYG1ErIuIHcBc4MKCNiOAJwEi4iWgRtJREfHniHguWf4OsAYYlGGsZmZWIMsEMQh4Pa/cxL4f8iuBSwAkjQGOA/Z6qrakGmAU8GzaRiRdI6lBUoMfVm5mVjpZJgilLIuC8p1ApaRG4DpgBbnTS7kVSEcAjwBfj4itaRuJiBkRURcRdVVVVSUJ3MzMoGeG624CBueVq4GN+Q2SD/2rACQJWJ+8kNSLXHJ4MCIezTBOMzNLkeUIYhlQK2mIpN7ABGBBfgNJ/ZM6gInAkojYmiSL+4E1EfG9DGM0M7MWZDaCiIhmSZOBhUAFMDMiVkualNRPB4YDcyTtBF4Erk66fwz4IrAqOf0EcFNEPJ5VvGZmtrf9JghJH2jp3L+kYyPiT/vrn3ygP16wbHre+2eA2pR+T5N+DSMbv5wC/7GqwzZnZlZSHzoRzr+z5Kstdopp8e43kp4sqJtf6mDMzKzzKHaKKf9b/ID91B3cMsi8ZmYHu2IjiGjhfVrZzMy6kGIjiA9K+ga50cLu9yRl/+jAzKwLK5Ygfgz0TXkP8JNMIjIzs05hvwkiIqZ2VCBmZta57PcahKSvSqpN3kvSTEl/TabnHtUxIZqZWTkUu0h9PfBa8v4K4GTgw8A3gB9kF5aZmZVbsQTRHBHvJ+8vAOZExOaI+DVweLahmZlZORVLELskHS2pDzAW+HVe3aHZhWVmZuVW7C6mW4AGcnMpLYiI1QCSzgbWZRybmZmVUbG7mH4u6Tigb0RsyatqAC7PNDIzMyurYpP1XZL3Pq2Jn9NgZtZFFTvF9DDQmLxg7/mXAicIM7Muq1iC+Cy5U0knAf8H+FlErM08KjMzK7v93sUUEfMiYgJwNvAq8F1JTycXqc3MrAtr7SNHtwN/BbaS+/1Dn8wiMjOzTqHYRepPkvsF9Rhyv4G4JyIaOiIwMzMrr2IjiCfJJYengUOAL0n6we5XsZVLOk/Sy5LWSpqSUl8paV4yt9NSSSPz6mZKekPSC23cJzMzK4FiF6mvOtAVS6oA7gPGAU3AMkkLIuLFvGY3AY0RcbGkYUn7sUndLOBeYM6BxmBmZgeu2A/lZrdUl/yAbn/GAGsjYl3Sfi5wIZCfIEYAdyTbeklSjaSjIuIvEbFEUk0r9sHMzDJQ9CK1pDMkXSrpg0n5JEk/JXfaaX8GAa/nlZuSZflWApck6x0DHAdUtzL23fFdI6lBUsOmTZva0tXMzPaj2PMg7gZmkvs9xC8k3Qr8CngWqC2y7rSfXhc+x/pOoFJSI3AdsAJoLh523gojZkREXUTUVVX5KahmZqVS7BrEZ4BREbFdUiWwETgpIl5pxbqbgMF55eqk/x4RsZXkOodyc3msT15mZlZmxU4xvRsR2wGSyfpebmVyAFgG1EoaIqk3MAFYkN9AUv+kDmAisCRJGmZmVmbFRhBDJeV/qNckZQEREeNb6hgRzZImAwvJTRc+MyJWS5qU1E8HhgNzJO0kd/H66t39Jf0MOAcYKKkJuDUi7m/zHpqZ2QFRROFlgbzK9Ck1dndQRPw2k6gOUF1dXTQ0+Hd8ZmatJWl5RNSl1RUbQfQHqiPivmRFS4EqcknixlIGaWZmnUuxaxD/zN7XDXoDdeRO/UzKKCYzM+sEio0gekdE/m8Zno6IzcBmSYdnGJeZmZVZsRFEZX4hIibnFf2jAzOzLqxYgnhW0lcLF0r6GrA0m5DMzKwzKHaK6QZgvqTPAc8ly04jN7PrRRnGZWZmZVZssr43gDMlfQo4IVn8i4h4KvPIzMysrIqNIABIEoKTgplZN9LaR46amVk34wRhZmapnCDMzCyVE4SZmaVygjAzs1ROEGZmlsoJwszMUjlBmJlZKicIMzNL5QRhZmapMk0Qks6T9LKktZKmpNRXSpon6XlJSyWNbG1fMzPLVmYJQlIFcB9wPjACuELSiIJmNwGNEXES8CXgnjb0NTOzDGU5ghgDrI2IdRGxA5gLXFjQZgTwJEBEvATUSDqqlX3NzCxDWSaIQUD+40qbkmX5VgKXAEgaAxwHVLeyL0m/ayQ1SGrYtGlTiUI3M7MsE4RSlkVB+U6gUlIjcB2wAmhuZd/cwogZEVEXEXVVVX4KqplZqbTqeRAHqAkYnFeuBjbmN4iIrcBVAJIErE9ehxXra2Zm2cpyBLEMqJU0RFJvYAKwIL+BpP5JHcBEYEmSNIr2NTOzbGU2goiIZkmTgYVABTAzIlZLmpTUTweGA3Mk7QReBK7eX9+sYjUzs30pIvXU/kGprq4uGhoayh2GmdlBQ9LyiKhLq/Mvqc3MLJUThJmZpXKCMDOzVE4QZmaWygnCzMxSOUGYmVkqJwgzM0vlBGFmZqmcIMzMLJUThJmZpXKCMDOzVE4QZmaWygnCzMxSOUGYmVkqJwgzM0vlBGFmZqmcIMzMLJUThJmZpco0QUg6T9LLktZKmpJS30/SY5JWSlot6aq8uuslvZAs/3qWcZqZ2b4ySxCSKoD7gPOBEcAVkkYUNLsWeDEiTgbOAb4rqbekkcBXgTHAycAFkmqzitXMzPaV5QhiDLA2ItZFxA5gLnBhQZsA+koScATwFtAMDAf+EBH/LyKagd8CF2cYq5mZFcgyQQwCXs8rNyXL8t1LLhlsBFYB10fELuAF4CxJR0o6DPh7YHCGsZqZWYGeGa5bKcuioFwPNAKfAoYCv5L0u4hYI+ku4FfANmAluZHFvhuRrgGuATj22GNLE7mZmWU6gmhi72/91eRGCvmuAh6NnLXAemAYQETcHxGnRsRZ5E49vZK2kYiYERF1EVFXVVVV8p0wM+uuskwQy4BaSUMk9QYmAAsK2vwJGAsg6Sjgo8C6pPzB5O+xwCXAzzKM1czMCmR2iikimiVNBhYCFcDMiFgtaVJSPx24HZglaRW5U1I3RsSbySoekXQk8D5wbURsySpWMzPbV5bXIIiIx4HHC5ZNz3u/ETi3hb6fyDI2MzPbP/+S2szMUjlBmJlZKicIMzNL5QRhZmapnCDMzCyVE4SZmaVygjAzs1ROEGZmlsoJwszMUjlBmJlZKicIMzNL5QRhZmapnCDMzCyVE4SZmaVygjAzs1ROEGZmlsoJwszMUjlBmJlZKicIMzNLlWmCkHSepJclrZU0JaW+n6THJK2UtFrSVXl1NyTLXpD0M0l9sozVzMz2llmCkFQB3AecD4wArpA0oqDZtcCLEXEycA7wXUm9JQ0C/gGoi4iRQAUwIatYzcxsX1mOIMYAayNiXUTsAOYCFxa0CaCvJAFHAG8BzUldT+BQST2Bw4CNGcZqZmYFskwQg4DX88pNybJ89wLDyX34rwKuj4hdEbEBmAb8Cfgz8NeIWJS2EUnXSGqQ1LBp06ZS74OZWbeVZYJQyrIoKNcDjcAxwCnAvZI+IKmS3GhjSFJ3uKQvpG0kImZERF1E1FVVVZUqdjOzbi/LBNEEDM4rV7PvaaKrgEcjZy2wHhgGfBpYHxGbIuJ94FHgzAxjNTOzAlkmiGVAraQhknqTu8i8oKDNn4CxAJKOAj4KrEuW/52kw5LrE2OBNRnGamZmBXpmteKIaJY0GVhI7i6kmRGxWtKkpH46cDswS9IqcqekboyIN4E3JT0MPEfuovUKYEZWsZqZ2b4UUXhZ4OBVV1cXDQ0N5Q7DzOygIWl5RNSl1fmX1GZmliqzU0xmZt3BrtiV+toZO4mI3F+Cnbt25urYxa5dub97tSn4u8868/oVbqNXj16cPfjsku+bE4TZQaItH0RBCx8yKX3b8kHU1g+wVvXfT79Wb7/Ih21btrnf9ez+oM/r1xkc2edIFl++uOTrdYIALnvsMt7b+V65w7AuLIgu8UFUShWqQBIVqqCHeuRe9KBHj+Sv0l/5/fbqn9K3Z4+e9Fbvlvu0Y5tCVPSo2G+/1P5JP5HEkWw7dRuFsbYQc68evTL5N3KCAIb2H8qOnTvKHYZ1UUF0jg+iIn1399/fB9F+P6Db2M86PycI4I5P3FHuEMzMOh3fxWRmZqmcIMzMLJUThJmZpXKCMDOzVE4QZmaWygnCzMxSOUGYmVkqJwgzM0vVpab7lrQJ+OMBdh8IvFnCcLo6H6+28fFqGx+vtmnP8TouIlKf19ylEkR7SGpoaU5025ePV9v4eLWNj1fbZHW8fIrJzMxSOUGYmVkqJ4i/8TOv28bHq218vNrGx6ttMjlevgZhZmapPIIwM7NUThBmZpaqWyUISedJelnSWklTUuorJc2T9LykpZJGliPOcmrFMRom6RlJ70n6p7b07YraebxmSnpD0gsdF3H5HegxkzRY0m8krZG0WtL1HRv5wUNSnaQftHs93eUahKQK4N+BcUATsAy4IiJezGtzN7AtIqZKGgbcFxFjyxJwGbTyGH0QOA64CNgSEdNa27erac/xSurOArYBcyKiW3wZaef/saOBoyPiOUl9geXARV35/1i5dacRxBhgbUSsi4gdwFzgwoI2I4AnASLiJaBG0lEdG2ZZFT1GEfFGRCwD3m9r3y6oPceLiFgCvNUhkXYeB3zMIuLPEfFc8v4dYA0wqGPC7niSaiS9JOknkl6Q9KCkT0v6vaRXJI1JXv9X0ork70eTvudI+nny/rZktLpY0jpJ/9DaGLpTghgEvJ5XbmLf/1wrgUsAJI0h9y2mukOi6xxac4yy6Huw6o773F4lOWaSaoBRwLOlCavTOh64BzgJGAZ8Dvg48E/ATcBLwFkRMQq4BfhOC+sZBtSTS9C3SurVmo33bFfoBxelLCs8v3YncI+kRmAVsAJozjiuzqQ1xyiLvger7rjP7dXuYybpCOAR4OsRsbUkUXVe6yNiFYCk1cCTERGSVgE1QD9gtqRacsexpQ/+X0TEe8B7kt4AjiKXnPerOyWIJmBwXrka2JjfIPnPdhWAJAHrk1d3UfQYZdT3YNUd97m92nXMkm++jwAPRsSjJY6tM3ov7/2uvPIucp/ftwO/iYiLk1HV4lasZyet/OzvTqeYlgG1koZI6g1MABbkN5DUP6kDmAgs6QbfUPIVPUYZ9T1Ydcd9bq8DPmbJl7b7gTUR8b0MYzyY9AM2JO+vLPXKu80IIiKaJU0GFgIVwMyIWC1pUlI/HRgOzJG0E3gRuLpsAZdBa46RpA8BDcAHgF2Svg6MiIitaX3LsiMdpATH62fAOcBASU3ArRFxfzn2paO055iROw//RWBVchoY4KaIeLyDd6Mz+Vdyp5i+ATxV6pV3m9tczcysbbrTKSYzM2sDJwgzM0vlBGFmZqmcIMzMLJUThJmZpXKCMOtgkv5F0qfLHYdZMb7N1SwDyY+6FBG7yh2L2YHyCMJsPyTdJem/5ZVvk/SPkv67pGXKPTtkalJXkzyr4H8AzwGDJc1KZuJcJemGpN0sSZcm78cmM3GuSmbcPCRZ/pqkqZKeS+qGdfzeW3fnBGG2f3OBy/PKlwGbgFpyM2OeApyWPNsB4KPknu8wChgIDIqIkRFxIvBv+SuW1AeYBVye1PcE/mtekzcj4lTgf5KbvdOsQzlBmO1HRKwAPijpGEknA1vITflwLrnZfp8jN5VybdLljxHxh+T9OuDDkn4o6TygcF6vj5KbrfPfk/Js4Ky8+t2T0S0nN3OnWYfqNnMxmbXDw8ClwIfIjShqgDsi4kf5jZLZNP9zdzkitiRJpR64ltzo4yv5XYpsd/cMnK2efdOslDyCMCtuLrlZRy8llywWAl9JnkuApEHJYzL3Imkg0CMiHgFuBk4taLL7qYXHJ+UvAr/NZhfM2s7fSsyKSGYb7QtsiIg/A3+WNBx4JnezEtuAL5D7pp9vEPBvknZ/EftmwXq3S7oKeEhST3JTYU/PcFfM2sS3uZqZWSqfYjIzs1ROEGZmlsoJwszMUjlBmJlZKicIMzNL5QRhZmapnCDMzCzV/wd/bu9V5tqgRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "uu_exp = mdf[mdf['algo'] == 'UU']\n",
    "sns.lineplot(x='version', y='GRMSE', hue='data', data=uu_exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ceaf3152",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-18T16:47:32.762389Z",
     "iopub.status.busy": "2021-06-18T16:47:32.759388Z",
     "iopub.status.idle": "2021-06-18T16:47:32.884015Z",
     "shell.execute_reply": "2021-06-18T16:47:32.885004Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='version', ylabel='nDCG'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcF0lEQVR4nO3df7RVZb3v8feHH0mdKNhCiWy8oO1ElATaoFnHqCTARqBpiXZLTC96jox79Ha6cR3+4jjqaFqp5VUxPWwbdrhaRthRwSyOWSlsEH8gekKk3MBRwo3C8Sf6vX+sCS2WC9Z69l5z/+LzGmONteYzn2fO55k6+Ow515zPUkRgZmZWrV6d3QEzM+teHBxmZpbEwWFmZkkcHGZmlsTBYWZmSfp0dgc6wqBBg2L48OGd3Q0zs25lxYoVf4mIwaXl+0RwDB8+nObm5s7uhplZtyLpT+XKfanKzMySODjMzCyJg8PMzJI4OMzMLImDw8zMkjg4zMwsiYPDzMyS7BPPcbTV3LtW8+TGlzu7G2ZmbTbqwPdxyecPr+k2fcZhZmZJfMaxF7VOaTOznsBnHGZmlsTBYWZmSRwcZmaWxMFhZmZJHBxmZpbEwWFmZkkcHGZmlsTBYWZmSXINDklTJD0taa2kOWXWS9K12frHJI3LyvtJWibpUUmrJc0tanOppA2SVmWv4/Mcg5mZ7S63J8cl9QauAyYBLcBySYsi4smialOBhux1FHB99v468OmI2C6pL/CgpHsi4qGs3fcj4qq8+r7LPXPgPx/PfTdmZrk5YDRMvbymm8zzjGMCsDYi1kXEG8ACYHpJnenArVHwEDBA0pBseXtWp2/2ihz7amZmVcpzrqqhwHNFyy0UziYq1RkKbMrOWFYAHwKui4iHi+rNlvRVoBn4ekS0lu5c0ixgFsBBBx3UthHUOKXNzHqCPM84VKas9Kxhj3Ui4q2IGAPUAxMkHZGtvx44BBgDbAK+W27nETEvIhojonHw4MHpvTczs7LyDI4WYFjRcj2wMbVORGwFlgJTsuXns1B5G7iJwiUxMzPrIHkGx3KgQdIISe8CZgCLSuosAr6a3V11NPBSRGySNFjSAABJ7waOA57KlocUtT8ReCLHMZiZWYncvuOIiB2SZgOLgd7ALRGxWtI52fobgLuB44G1wCvAGVnzIUBT9j1HL+D2iPhltu47ksZQuKS1Hjg7rzGYmdk7KaLn36zU2NgYzc3Nnd0NM7NuRdKKiGgsLfeT42ZmlsTBYWZmSRwcZmaWxMFhZmZJHBxmZpbEwWFmZkkcHGZmlsTBYWZmSRwcZmaWxMFhZmZJHBxmZpbEwWFmZkkcHGZmlsTBYWZmSRwcZmaWxMFhZmZJHBxmZpbEwWFmZkkcHGZmlsTBYWZmSRwcZmaWxMFhZmZJHBxmZpbEwWFmZkkcHGZmlsTBYWZmSRwcZmaWxMFhZmZJHBxmZpYk1+CQNEXS05LWSppTZr0kXZutf0zSuKy8n6Rlkh6VtFrS3KI2dZLuk/TH7H1gnmMwM7Pd5RYcknoD1wFTgVHAqZJGlVSbCjRkr1nA9Vn568CnI+JIYAwwRdLR2bo5wP0R0QDcny2bmVkHyfOMYwKwNiLWRcQbwAJgekmd6cCtUfAQMEDSkGx5e1anb/aKojZN2ecm4IQcx2BmZiXyDI6hwHNFyy1ZWVV1JPWWtAp4AbgvIh7O6nwwIjYBZO8fKLdzSbMkNUtq3rx5c3vHYmZmmTyDQ2XKoto6EfFWRIwB6oEJko5I2XlEzIuIxohoHDx4cEpTMzPbizyDowUYVrRcD2xMrRMRW4GlwJSs6HlJQwCy9xdq1mMzM6soz+BYDjRIGiHpXcAMYFFJnUXAV7O7q44GXoqITZIGSxoAIOndwHHAU0VtTs8+nw78IscxmJlZiT55bTgidkiaDSwGegO3RMRqSedk628A7gaOB9YCrwBnZM2HAE3ZnVm9gNsj4pfZusuB2yWdCfwZ+GJeYzAzs3dSROnXDj1PY2NjNDc3d3Y3zMy6FUkrIqKxtNxPjpuZWRIHh5mZJXFwmJlZEgeHmZklcXCYmVkSB4eZmSVxcJiZWRIHh5mZJXFwmJlZEgeHmZklcXCYmVmS3CY5NDPrCt58801aWlp47bXXOrsrXVa/fv2or6+nb9++VdV3cJhZj9bS0kL//v0ZPnw4Urnfjtu3RQRbtmyhpaWFESNGVNXGl6rMrEd77bXX2H///R0aeyCJ/fffP+mMzMFhZj2eQ2PvUo+Pg8PMLAeXXnopV1111R7XL1y4kCeffLIDe1Q7Dg4zs07g4DAzM771rW9x6KGHctxxx/H0008DcNNNNzF+/HiOPPJITjrpJF555RV+//vfs2jRIr7xjW8wZswYnnnmmbL1uioHh5lZDaxYsYIFCxbwyCOPcOedd7J8+XIAvvCFL7B8+XIeffRRDjvsMG6++WaOOeYYpk2bxpVXXsmqVas45JBDytbrqnw7rplZDfz2t7/lxBNP5D3veQ8A06ZNA+CJJ57gwgsvZOvWrWzfvp3JkyeXbV9tva7AwWFmViPl7k6aOXMmCxcu5Mgjj2T+/PksXbq0bNtq63UFvlRlZlYDxx57LD//+c959dVX2bZtG3fddRcA27ZtY8iQIbz55pvcdtttu+r379+fbdu27VreU72uyMFhZlYD48aN45RTTmHMmDGcdNJJ/O3f/i0Al112GUcddRSTJk1i5MiRu+rPmDGDK6+8krFjx/LMM8/ssV5XpIjo7D7krrGxMZqbmzu7G2bWCdasWcNhhx3W2d3o8sodJ0krIqKxtK7POMzMLImDw8zMkjg4zMwsiYPDzMySODjMzCxJrsEhaYqkpyWtlTSnzHpJujZb/5ikcVn5MEm/kbRG0mpJ/1DU5lJJGyStyl7H5zkGMzPbXW7BIak3cB0wFRgFnCppVEm1qUBD9poFXJ+V7wC+HhGHAUcD55a0/X5EjMled+c1BjOzjjJ//nxmz54NwAMPPMC4cePo06cPP/3pT3er19TURENDAw0NDTQ1Ne0qf/bZZznqqKNoaGjglFNO4Y033gAqT+/eFnsNDknTJZ1btPywpHXZ6+QK254ArI2IdRHxBrAAmF5SZzpwaxQ8BAyQNCQiNkXESoCI2AasAYYmjs3MrFs66KCDmD9/Pqeddtpu5S+++CJz587l4YcfZtmyZcydO5fW1lYAvvnNb3L++efzxz/+kYEDB+Y6SWKlM47/DSwqWt4PGA9MBP6uQtuhwHNFyy288x//inUkDQfGAg8XFc/OLm3dImlghX6YmXWq9evXM3LkSM466yyOOOIIvvzlL/OrX/2Kj3/84zQ0NLBs2bLd6g8fPpyPfOQj9Oq1+z/RixcvZtKkSdTV1TFw4EAmTZrEvffeS0Tw61//mpNPLvw9f/rpp7Nw4cJ39OOmm25i6tSpvPrqq+0aT6VJDt8VEcX/sD8YEVuALZL+pkLbcr9FWPqY+l7rSHov8DPgvIh4OSu+Hrgsq3cZ8F3ga+/YuTSLwuUvDjrooApdNbN9wdy7VvPkxpcrV0ww6sD3ccnnD69Yb+3atdxxxx3MmzeP8ePH85Of/IQHH3yQRYsW8e1vf5sTTjih4jY2bNjAsGHDdi3X19ezYcMGtmzZwoABA+jTp89u5cV++MMfsmTJEhYuXMh+++2XNsgSlYJjt7/mI2J20eLgCm1bgGFFy/XAxmrrSOpLITRui4g7i/rw/M7Pkm4Cfllu5xExD5gHhSlHKvTVzCxXI0aMYPTo0QAcfvjhfOYzn0ESo0ePZv369VVto9wUUZL2WL7Tj3/8Y+rr61m4cCF9+/Zt2wCKVAqOhyX9j4i4qaRDZwPL9tBmp+VAg6QRwAZgBnBaSZ1FFC47LQCOAl6KiE0qjPhmYE1EfK9k30MiYlO2eCLwRIV+mJkBVHVmkJfiv/J79eq1a7lXr17s2LGjqm3U19fvNt16S0sLEydOZNCgQWzdupUdO3bQp08fWlpaOPDAA3fVO+KII1i1ahUtLS2MGDGi3WOp9B3H+cAZ2a2x381eS4GZwHl7axgRO4DZwGIKX27fHhGrJZ0j6Zys2t3AOmAtcBPw91n5x4GvAJ8uc9vtdyQ9Lukx4FNZH83MerzJkyezZMkSWltbaW1tZcmSJUyePBlJfOpTn9p1B1ZTUxPTp//1XqSxY8dy4403Mm3aNDZuLL3wk26vZxwR8QJwjKRPAzuj+t8i4tfVbDy7VfbukrIbij4HcG6Zdg9S/vsPIuIr1ezbzKy7Wr58OSeeeCKtra3cddddXHLJJaxevZq6ujouuugixo8fD8DFF19MXV0dAFdccQUzZszgwgsvZOzYsZx55pm7bfMTn/gEV111FZ/73Oe47777GDRoUJv7t9dp1SWNBwZFxD0l5Z8HNkbEijbvuQN5WnWzfZenVa9OLadVv5LCZaZ37CNbZ2Zm+5hKwbF/RKwvLYyItcD+ufTIzMy6tErB8e69rKv0HIeZmfVAlYLjV5K+peIbggFJc4GqviA3M7OepdJzHF8HfgSslbQqKxtD4RmNs/LrlpmZdVWVbsf9Lwqz2h7MX2/HXR0R63LvmZmZdUkVp1WX1IdCaIzMXqOyMjMzq5Fqp1XvCipNq34gsJrCJasDKcxc+w1gdbbOzMxqbE/TqncVlc44vg1cHxETI+L8iDgvIj5J4Qea/jn/7pmZdX+1mlZ96dKlfPKTn+RLX/oSH/7wh5kzZw633XYbEyZMYPTo0TzzzDMdMp5Kl5yOjoiZpYURca2kp/PpkplZTu6ZA//5eG23ecBomHp5xWq1mFYd4NFHH2XNmjXU1dVx8MEHc9ZZZ7Fs2TKuueYafvCDH3D11Ve3bzxVqHTGsbdf+3illh0xM+vJdk6r3qtXrzZPqw4wfvx4hgwZwn777cchhxzCZz/7WYDk7bRHpTOO90v6QplyAe/LoT9mZvmp4swgL7WYVr2W22mPSsHx78Dni5Z3zoiobJ2Zme1jKj3HcQaApH7AScDwojb+VT0zsxzsaVr1rmKv06rvqiTdC2wFVgJv7SyPiO/m1rMa8rTqZvsuT6tenZRp1at9kK8+IqbUonNmZta9VXxyPPN7SaNz7YmZmXUL1Z5xfAKYKelZ4HUKX45HRHwkt56ZmVmXVG1wTM21F2Zm1m1UFRwR8ae8O2JmZt1Dtd9xmJmZAQ4OM7Muodpp1ZuammhoaKChoYGmpqbO6GrV33GYmVkH2Tmt+lVXXbVb+YsvvsjcuXNpbm5GEh/96EeZNm0aAwcO7ND++YzDzCxntZpWffHixUyaNIm6ujoGDhzIpEmTuPfee3e1ueCCC/jYxz5GY2MjK1euZPLkyRxyyCHccMMNNR2PzzjMbJ9xxbIreOrFp2q6zZF1I/nmhG9WrFeLadU3bNjAsGHDdi3X19ezYcOGXcvDhg3jD3/4A+effz4zZ87kd7/7Ha+99hqHH34455xzTpvGV46Dw8ysA+ycVh1o87Tq5aaIkrTr87Rp04DCFOvbt2+nf//+9O/fn379+rF161YGDBjQ7nGAg8PM9iHVnBnkpRbTodfX17N06dJdyy0tLUycOPEd+yjefuo+quHvOMzMuonJkyezZMkSWltbaW1tZcmSJUyePLnD++HgMDPrYpYvX059fT133HEHZ599NocffjgAdXV1XHTRRYwfP57x48dz8cUXU1dX1+H9q2pa9TZvXJoCXAP0Bn4UEZeXrFe2/ngKP0U7MyJWShoG3AocALwNzIuIa7I2dcD/o/DbIOuBL0VE69764WnVzfZdnla9OinTqud2xiGpN3AdhXmuRgGnShpVUm0q0JC9ZgHXZ+U7gK9HxGHA0cC5RW3nAPdHRANwf7ZsZmYdJM9LVROAtRGxLiLeABYA00vqTAdujYKHgAGShkTEpohYCRAR24A1wNCiNjsfl2wCTshxDGZmViLP4BgKPFe03MJf//Gvuo6k4cBY4OGs6IMRsQkge/9AuZ1LmiWpWVLz5s2b2zoGMzMrkWdwqExZ6Rcqe60j6b3Az4DzIuLllJ1HxLyIaIyIxsGDB6c0NbMeJs/vcnuC1OOTZ3C0AMOKluuBjdXWkdSXQmjcFhF3FtV5XtKQrM4Q4IUa99vMepB+/fqxZcsWh8ceRARbtmyhX79+VbfJ8wHA5UCDpBHABmAGcFpJnUXAbEkLgKOAlyJiU3a31c3Amoj4Xpk2pwOXZ++/yHEMZtbN1dfX09LSgi9Z71m/fv2or6+vun5uwREROyTNBhZTuB33lohYLemcbP0NwN0UbsVdS+F23DOy5h8HvgI8LmlVVnZBRNxNITBul3Qm8Gfgi3mNwcy6v759+zJixIjO7kaPkutzHF2Fn+MwM0vX4c9xmJlZz+TgMDOzJA4OMzNL4uAwM7MkDg4zM0vi4DAzsyQODjMzS+LgMDOzJA4OMzNL4uAwM7MkDg4zM0vi4DAzsyQODjMzS+LgMDOzJA4OMzNL4uAwM7MkDg4zM0vi4DAzsyQODjMzS+LgMDOzJA4OMzNL4uAwM7MkDg4zM0vi4DAzsyQODjMzS+LgMDOzJA4OMzNL4uAwM7MkDg4zM0uSa3BImiLpaUlrJc0ps16Srs3WPyZpXNG6WyS9IOmJkjaXStogaVX2Oj7PMZiZ2e5yCw5JvYHrgKnAKOBUSaNKqk0FGrLXLOD6onXzgSl72Pz3I2JM9rq7ph03M7O9yvOMYwKwNiLWRcQbwAJgekmd6cCtUfAQMEDSEICIeAB4Mcf+mZlZG+QZHEOB54qWW7Ky1DrlzM4ubd0iaWC5CpJmSWqW1Lx58+aUfpuZ2V7kGRwqUxZtqFPqeuAQYAywCfhuuUoRMS8iGiOicfDgwRU2aWZm1cozOFqAYUXL9cDGNtTZTUQ8HxFvRcTbwE0ULomZmVkHyTM4lgMNkkZIehcwA1hUUmcR8NXs7qqjgZciYtPeNrrzO5DMicATe6prZma11yevDUfEDkmzgcVAb+CWiFgt6Zxs/Q3A3cDxwFrgFeCMne0l/SswERgkqQW4JCJuBr4jaQyFS1rrgbPzGoOZmb2TIip9pdD9NTY2RnNzc2d3w8ysW5G0IiIaS8v95LiZmSVxcJiZWRIHh5mZJXFwmJlZEgeHmZklcXCYmVkSB4eZmSVxcJiZWRIHh5mZJXFwmJlZEgeHmZklcXCYmVkSB4eZmSVxcJiZWRIHh5mZJXFwmJlZEgeHmZklcXCYmVkSB4eZmSVxcJiZWRIHh5mZJXFwmJlZEgeHmZklcXCYmVkSB4eZmSXp09kd6MpufPRG7nn2ns7uhvVgQVQsi3hnnaq3VaZtNdsvt62y+yxpW814kvYZleu0a/ttbFfNPqsZT7Xbb89/o6s/dTXHHHhMVXWr5eDYi0HvHsTBAw7u7G5YDyf0zjKpcp0yZeWLarf90nblVNuuPfXavP2SsmrGs8dtVbHPto6nXNu2/jc64G8OqGp/KRwce3HSh0/ipA+f1NndMDPrUvwdh5mZJXFwmJlZklyDQ9IUSU9LWitpTpn1knRttv4xSeOK1t0i6QVJT5S0qZN0n6Q/Zu8D8xyDmZntLrfgkNQbuA6YCowCTpU0qqTaVKAhe80Cri9aNx+YUmbTc4D7I6IBuD9bNjOzDpLnGccEYG1ErIuIN4AFwPSSOtOBW6PgIWCApCEAEfEA8GKZ7U4HmrLPTcAJeXTezMzKyzM4hgLPFS23ZGWpdUp9MCI2AWTvHyhXSdIsSc2Smjdv3pzUcTMz27M8g6PczcqlT6xUU6dNImJeRDRGROPgwYNrsUkzMyPf4GgBhhUt1wMb21Cn1PM7L2dl7y+0s59mZpYgzwcAlwMNkkYAG4AZwGkldRYBsyUtAI4CXtp5GWovFgGnA5dn77+o1JEVK1b8RdKfEvu/0yDgL21suy/y8Urj45XGxytde47ZfytXqGrnwWkLSccDVwO9gVsi4luSzgGIiBtUeDb+hxTunnoFOCMimrO2/wpMpDDo54FLIuJmSfsDtwMHAX8GvhgR5b5Er9UYmiOiMa/t9zQ+Xml8vNL4eKXL45jlOuVIRNwN3F1SdkPR5wDO3UPbU/dQvgX4TA27aWZmCfzkuJmZJXFwVDavszvQzfh4pfHxSuPjla7mxyzX7zjMzKzn8RmHmZklcXCYmVkSB0emipl8B0r6eTaL7zJJR3RGPztTFcdopKQ/SHpd0j+mtO2J2nm8ys4O3ZO19XhJGibpN5LWSFot6R86tufdh6RGSde2ezv+jmPXTL7/AUyi8DT7cuDUiHiyqM6VwPaImCtpJHBdROwztwVXeYw+QOGBoROA1oi4qtq2PU17jle27lhgO4VJQHv8Hynt/P9rCDAkIlZK6g+sAE7oyf9/dTafcRRUM5PvKArTuBMRTwHDJX2wY7vZqSoeo4h4ISKWA2+mtu2B2nO89jY7dE/V5uMVEZsiYmX2eRuwhsqTpXZbkoZLekrSjyQ9Iek2ScdJ+l32O0UTstfvJT2SvR+atZ0o6ZfZ50uzM9ulktZJ+p/V9sHBUVDNLL2PAl8AkDSBwl8+9R3Su66hLTMZ16Jtd7Uvjrk9anK8JA0HxgIP16ZbXdaHgGuAjwAjKUzn9AngH4ELgKeAYyNiLHAx8O09bGckMJlCcF8iqW81O8/1yfFupJpZei8HrpG0CngceATYkXO/upL2zGSc2yzIXdi+OOb2aPfxkvRe4GfAeRHxck161XU9GxGPA0haTeHH7ULS48Bw4P1Ak6QGCsdxT4HwbxHxOvC6pBeAD1II7b1ycBRUnKU3+x/xDCj85C3wbPbaV7RlJuNatO2u9sUxt0e7jlf2l/LPgNsi4s4a960rer3o89tFy29T+Hf9MuA3EXFidha2tIrtvEWVmeBLVQW7ZvKV9C4KM/kuKq4gaUC2DuAs4IF94K+aYhWPUU5tu6t9cczt0ebjlf0hdzOwJiK+l2Mfu5P3U5iVHGBmrTfuMw4gInZImg0s5q8z+a5W0Uy+wGHArZLeAp4Ezuy0DneCao6RpAOAZuB9wNuSzgNGRcTL5dp2ykA6SA2O167ZoSW1kM0O3Rlj6QjtOV4UrvN/BXg8u5QMcEE2yeq+6jsULlX9L+DXtd64b8c1M7MkvlRlZmZJHBxmZpbEwWFmZkkcHGZmlsTBYWZmSRwcZl2EpH+SdFxn98OsEt+Oa9aBsofVFBFvd3ZfzNrKZxxmbSDpCkl/X7R8qaSvS/qGpOUq/G7L3Gzd8Oy3Iv4vsBIYJml+NrPp45LOz+rNl3Ry9vkz2cymj2czmO6Xla+XNFfSymzdyI4fve3rHBxmbbMAOKVo+UvAZqCBwkyjY4CPZr+rAXAohd/WGAsMAoZGxBERMRr4l+INS+oHzAdOydb3Af6uqMpfImIccD2F2VDNOpSDw6wNIuIR4AOSDpR0JNBKYeqLz1KYOXklhSmrG7Imf4qIh7LP64CDJf1A0hSgdM6zQynMfvof2XITcGzR+p2T+K2gMBOqWYfyXFVmbfdT4GTgAApnIMOBf46IG4srZbOT/tfO5YhozcJmMnAuhbOVrxU3qbDfnTOaVj2bqVkt+YzDrO0WUJjF9WQKIbIY+Fr2uxBIGpr93OluJA0CekXEz4CLgHElVXb+wuSHsuWvAP+ezxDM0vmvFbM2ymZv7Q9siIhNwCZJhwF/KNw8xXbgv1M4Myg2FPgXSTv/cPs/Jdt9TdIZwB2S+lCYcvyGHIdilsS345qZWRJfqjIzsyQODjMzS+LgMDOzJA4OMzNL4uAwM7MkDg4zM0vi4DAzsyT/H4YSs46awwUEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "uu_exp = mdf[mdf['algo'] == 'UU']\n",
    "sns.lineplot(x='version', y='nDCG', hue='data', data=uu_exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa32180d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}