incremental-model-selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Incremental Model Selection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "import dask\n",
    "from dask.distributed import Client\n",
    "from dask_ml.datasets import make_classification\n",
    "from dask_ml.model_selection._incremental import fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://127.0.0.1:41941\n",
       "  <li><b>Dashboard: </b><a href='http://127.0.0.1:8787/status' target='_blank'>http://127.0.0.1:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>4</li>\n",
       "  <li><b>Memory: </b>16.68 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://127.0.0.1:41941' processes=4 cores=4>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Client(processes=True)\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_classification(n_samples=5000000, n_features=20,\n",
    "                           chunks=100000)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Incremental.fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_classification(n_samples=5000000, n_features=20,\n",
    "                           chunks=100000)\n",
    "model = SGDClassifier(tol=1e-3, penalty='elasticnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_iter = 100\n",
    "patience = 10\n",
    "tol = 0.001\n",
    "\n",
    "def update(info):\n",
    "    [info] = info.values()\n",
    "    if max_iter is not None and len(info) > max_iter:\n",
    "        return {0: 0}\n",
    "\n",
    "    if len(info) > patience:\n",
    "        old = info[-patience]['score']\n",
    "        if all(d['score'] < old + tol for d in info[-patience:]):\n",
    "            return {0: 0}\n",
    "\n",
    "    return {0: 1}\n",
    "\n",
    "from dask_ml.model_selection._incremental import fit\n",
    "X_test = X.blocks[-1]\n",
    "X = X.blocks[:-1]\n",
    "y_test = y.blocks[-1]\n",
    "y = y.blocks[:-1]\n",
    "info, models, history = fit(model, [{}], X, y, X_test, y_test, update,\n",
    "                            {'classes': [0, 1]})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: [{'ident': 0, 'params': {}, 'score': 0.52541, 'time_step': 0},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.50198, 'time_step': 1},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53009, 'time_step': 2},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.51285, 'time_step': 3},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53274, 'time_step': 4},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.51814, 'time_step': 5},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.5239, 'time_step': 6},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53226, 'time_step': 7},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.52525, 'time_step': 8},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.52155, 'time_step': 9},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.52643, 'time_step': 10},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.51986, 'time_step': 11},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53414, 'time_step': 12},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53035, 'time_step': 13},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53451, 'time_step': 14},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.5343, 'time_step': 15},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53683, 'time_step': 16},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.52794, 'time_step': 17},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53375, 'time_step': 18},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53174, 'time_step': 19},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.52983, 'time_step': 20},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53063, 'time_step': 21},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.54107, 'time_step': 22},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53116, 'time_step': 23},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53947, 'time_step': 24},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53822, 'time_step': 25},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53825, 'time_step': 26},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.54315, 'time_step': 27},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.52376, 'time_step': 28},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53883, 'time_step': 29},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.54, 'time_step': 30},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53251, 'time_step': 31},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53648, 'time_step': 32},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53626, 'time_step': 33},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53865, 'time_step': 34},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53446, 'time_step': 35},\n",
       "  {'ident': 0, 'params': {}, 'score': 0.53912, 'time_step': 36}]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RandomSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 1, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 1, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 1, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 1, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 1, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 1, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 1, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 1}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 0}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 0}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 0}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 0}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 1, 7: 0, 8: 0, 9: 0}\n",
      "{0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0}\n"
     ]
    }
   ],
   "source": [
    "max_iter = 100\n",
    "patience = 10\n",
    "tol = 0.001\n",
    "\n",
    "def update(info):\n",
    "    out = {}\n",
    "    for ident, records in info.items():\n",
    "        if max_iter is not None and len(records) > max_iter:\n",
    "            out[ident] = 0\n",
    "\n",
    "        elif len(records) > patience:\n",
    "            old = records[-patience]['score']\n",
    "            if all(d['score'] < old + tol for d in records[-patience:]):\n",
    "                out[ident] = 0\n",
    "            else:\n",
    "                out[ident] = 1\n",
    "        \n",
    "        else:\n",
    "            out[ident] = 1\n",
    "    print(out)\n",
    "    return out\n",
    "\n",
    "X, y = make_classification(n_samples=5000000, n_features=20,\n",
    "                           chunks=100000)\n",
    "model = SGDClassifier(tol=1e-3, penalty='elasticnet')\n",
    "\n",
    "params = {'alpha': np.logspace(-2, 1, num=1000),\n",
    "          'l1_ratio': np.linspace(0, 1, num=1000),\n",
    "          'average': [True, False]}\n",
    "\n",
    "from sklearn.model_selection import ParameterSampler\n",
    "params_list = list(ParameterSampler(params, 10))\n",
    "\n",
    "from dask_ml.model_selection._incremental import fit\n",
    "X_test = X.blocks[-1]\n",
    "X = X.blocks[:-1]\n",
    "y_test = y.blocks[-1]\n",
    "y = y.blocks[:-1]\n",
    "info, models, history = fit(model, params_list, X, y, X_test, y_test, update,\n",
    "        {'classes': [0, 1]})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65046,\n",
       "  'time_step': 0},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65059,\n",
       "  'time_step': 1},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65083,\n",
       "  'time_step': 2},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65059,\n",
       "  'time_step': 3},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65083,\n",
       "  'time_step': 4},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65077,\n",
       "  'time_step': 5},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65069,\n",
       "  'time_step': 6},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65076,\n",
       "  'time_step': 7},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65074,\n",
       "  'time_step': 8},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65047,\n",
       "  'time_step': 9},\n",
       " {'ident': 1,\n",
       "  'params': {'alpha': 0.041842885079015804,\n",
       "   'average': False,\n",
       "   'l1_ratio': 0.22122122122122123},\n",
       "  'score': 0.65071,\n",
       "  'time_step': 10}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best = max(info, key=lambda k: info[k][-1]['score'])\n",
    "info[best]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Successive halving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{7: 1, 22: 1, 45: 1, 2: 1, 31: 1, 46: 1, 66: 1, 54: 1, 3: 1, 1: 1, 49: 1, 57: 1, 72: 1, 48: 1, 14: 1, 20: 1, 10: 1, 91: 1, 88: 1, 85: 1, 36: 1, 40: 1, 71: 1, 24: 1, 6: 1, 39: 1, 33: 1, 78: 1, 87: 1, 97: 1, 74: 1, 86: 1, 73: 1}\n",
      "{54: 1, 49: 1, 3: 1, 31: 1, 14: 1, 1: 1, 66: 1, 10: 1, 45: 1, 72: 1, 7: 1, 46: 1, 22: 1, 91: 1, 88: 1, 2: 1, 20: 1, 48: 1, 57: 1, 85: 1, 36: 1, 40: 1, 71: 1, 24: 1, 6: 1}\n",
      "{49: 1, 7: 1, 10: 1, 46: 1, 72: 1, 1: 1, 3: 1, 85: 1, 14: 1, 22: 1, 66: 1, 57: 1, 91: 1, 2: 1, 48: 1, 20: 1, 31: 1, 54: 1, 88: 1, 45: 1}\n",
      "{88: 1, 49: 1, 45: 1, 7: 1, 1: 1, 54: 1, 3: 1, 31: 1, 20: 1, 72: 1, 22: 1, 14: 1, 46: 1, 10: 1, 48: 1, 66: 1}\n",
      "{49: 1, 31: 1, 14: 1, 22: 1, 10: 1, 1: 1, 46: 1, 54: 1, 72: 1, 20: 1, 3: 1, 45: 1, 66: 1, 7: 1}\n",
      "{14: 1, 1: 1, 31: 1, 49: 1, 22: 1, 10: 1, 54: 1, 72: 1, 7: 1, 46: 1, 3: 1, 66: 1}\n",
      "{72: 1, 3: 1, 22: 1, 1: 1, 49: 1, 7: 1, 14: 1, 10: 1, 46: 1, 31: 1, 66: 1}\n",
      "{72: 1, 22: 1, 49: 1, 1: 1, 10: 1, 14: 1, 3: 1, 7: 1, 46: 1, 31: 1}\n",
      "{31: 1, 72: 1, 14: 1, 3: 1, 46: 1, 1: 1, 7: 1, 10: 1, 22: 1}\n",
      "{22: 1, 10: 1, 14: 1, 1: 1, 46: 1, 72: 1, 31: 1, 3: 1}\n",
      "{3: 1, 1: 1, 10: 1, 72: 1, 22: 1, 46: 1, 31: 1}\n",
      "{31: 2, 1: 2, 3: 2, 22: 2, 10: 2, 46: 2}\n",
      "{1: 2, 10: 2, 22: 2, 31: 2, 46: 2}\n",
      "{31: 4, 22: 4, 46: 4, 1: 4}\n",
      "{31: 5, 22: 5, 1: 5}\n",
      "{1: 8, 22: 8}\n"
     ]
    }
   ],
   "source": [
    "import toolz\n",
    "\n",
    "start = 100\n",
    "\n",
    "def inverse(start, batch):\n",
    "    \"\"\" Decrease target number of models inversely with time \"\"\"\n",
    "    return int(start / (1 + batch))\n",
    "\n",
    "\n",
    "def update(info):\n",
    "    example = toolz.first(info.values())\n",
    "    time_step = example[-1]['time_step']\n",
    "    \n",
    "    current_time_step = time_step + 1\n",
    "    next_time_step = current_time_step\n",
    "    while inverse(start, current_time_step) == inverse(start, next_time_step):\n",
    "        next_time_step += 1\n",
    "\n",
    "    \n",
    "    target = inverse(start, next_time_step)\n",
    "    best = toolz.topk(target, info, key=lambda k: info[k][-1]['score'])\n",
    "    \n",
    "    if len(best) == 1:\n",
    "        [best] = best\n",
    "        return {best: 0}\n",
    "    \n",
    "    out = {}\n",
    "    for k in best:\n",
    "        out[k] = next_time_step - current_time_step\n",
    "        \n",
    "    print(out)\n",
    "    return out\n",
    "\n",
    "model = SGDClassifier(tol=1e-3, penalty='elasticnet')\n",
    "\n",
    "params = {'alpha': np.logspace(-2, 1, num=1000),\n",
    "          'l1_ratio': np.linspace(0, 1, num=1000),\n",
    "          'average': [True, False]}\n",
    "params = {'alpha': np.logspace(-2, 1, num=1000),\n",
    "          'l1_ratio': np.linspace(0, 1, num=1000),\n",
    "          'average': [True, False]}\n",
    "\n",
    "from sklearn.model_selection import ParameterSampler\n",
    "params_list = list(ParameterSampler(params, start))\n",
    "\n",
    "from dask_ml.model_selection._incremental import fit\n",
    "X_test = X.blocks[-1]\n",
    "X = X.blocks[:-1]\n",
    "y_test = y.blocks[-1]\n",
    "y = y.blocks[:-1]\n",
    "info, models, history = fit(model, params_list, X, y, X_test, y_test, update,\n",
    "                            {'classes': [0, 1]})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: [{'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65365,\n",
       "   'time_step': 0},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65383,\n",
       "   'time_step': 1},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65386,\n",
       "   'time_step': 2},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65383,\n",
       "   'time_step': 3},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65385,\n",
       "   'time_step': 4},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65399,\n",
       "   'time_step': 5},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65398,\n",
       "   'time_step': 6},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65383,\n",
       "   'time_step': 7},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65384,\n",
       "   'time_step': 8},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.6539,\n",
       "   'time_step': 9},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65396,\n",
       "   'time_step': 10},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65395,\n",
       "   'time_step': 11},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65392,\n",
       "   'time_step': 13},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65383,\n",
       "   'time_step': 15},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65396,\n",
       "   'time_step': 19},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65395,\n",
       "   'time_step': 24},\n",
       "  {'ident': 1,\n",
       "   'params': {'alpha': 0.23570694139967277,\n",
       "    'average': False,\n",
       "    'l1_ratio': 0.03903903903903904},\n",
       "   'score': 0.65391,\n",
       "   'time_step': 32}]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "info"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}