Instantly share code, notes, and snippets.
-
Save microprediction/ef9d845ba18fb9c94f89871ef1cc7be1 to your computer and use it in GitHub Desktop.
navigation.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "nbformat": 4, | |
| "nbformat_minor": 0, | |
| "metadata": { | |
| "colab": { | |
| "name": "navigation.ipynb", | |
| "provenance": [], | |
| "authorship_tag": "ABX9TyNWQRO61xWrftBvlntSCveP", | |
| "include_colab_link": true | |
| }, | |
| "kernelspec": { | |
| "name": "python3", | |
| "display_name": "Python 3" | |
| } | |
| }, | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/microprediction/ef9d845ba18fb9c94f89871ef1cc7be1/navigation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "Nl1V2oCzTIgq", | |
| "outputId": "1cbad149-a7ad-4db0-838a-d78156318132", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 1000 | |
| } | |
| }, | |
| "source": [ | |
| "!pip install microprediction" | |
| ], | |
| "execution_count": 1, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Collecting microprediction\n", | |
| "\u001b[?25l Downloading https://files.pythonhosted.org/packages/1e/9c/5cc2a2a4fac7a6a52202fd8ed40c1b011916d033435a982ff3798ca4b843/microprediction-0.15.4-py3-none-any.whl (58kB)\n", | |
| "\r\u001b[K |█████▋ | 10kB 13.3MB/s eta 0:00:01\r\u001b[K |███████████▎ | 20kB 1.5MB/s eta 0:00:01\r\u001b[K |█████████████████ | 30kB 1.8MB/s eta 0:00:01\r\u001b[K |██████████████████████▌ | 40kB 2.1MB/s eta 0:00:01\r\u001b[K |████████████████████████████▏ | 51kB 1.9MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 61kB 1.8MB/s \n", | |
| "\u001b[?25hRequirement already satisfied: pathlib in /usr/local/lib/python3.6/dist-packages (from microprediction) (1.0.1)\n", | |
| "Collecting contexttimer\n", | |
| " Downloading https://files.pythonhosted.org/packages/1d/e0/504aa08a83dc2ff90f61a83b5f70d689e1f5138ab30576124ea2ff9f5076/contexttimer-0.3.3.tar.gz\n", | |
| "Collecting apscheduler\n", | |
| "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f3/34/9ef20ed473c4fd2c3df54ef77a27ae3fc7500b16b192add4720cab8b2c09/APScheduler-3.6.3-py2.py3-none-any.whl (58kB)\n", | |
| "\u001b[K |████████████████████████████████| 61kB 2.8MB/s \n", | |
| "\u001b[?25hCollecting tdigest\n", | |
| " Downloading https://files.pythonhosted.org/packages/32/72/f420480118cbdd18eb761b9936f0a927957130659a638449575b4a4f0aa7/tdigest-0.5.2.2-py2.py3-none-any.whl\n", | |
| "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from microprediction) (1.18.5)\n", | |
| "Requirement already satisfied: hyperopt in /usr/local/lib/python3.6/dist-packages (from microprediction) (0.1.2)\n", | |
| "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from microprediction) (1.1.2)\n", | |
| "Collecting genson\n", | |
| " Downloading https://files.pythonhosted.org/packages/e1/71/fbd2f1ad9695c92ad756b91f5a570f809c4959d3bd82266788cf222e5c5c/genson-1.2.2.tar.gz\n", | |
| "Collecting pycoingecko\n", | |
| " Downloading https://files.pythonhosted.org/packages/cf/b6/abf90df392a36fad22c86dfd5d089e2b9ea748a08ca54004113572cf51b0/pycoingecko-1.4.0-py3-none-any.whl\n", | |
| "Collecting getjson\n", | |
| " Downloading https://files.pythonhosted.org/packages/70/f0/538ef8ccaa16310a7e3ad27538277e2301796fd52ae49fde314f526ee925/getjson-1.0.0-py3-none-any.whl\n", | |
| "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from microprediction) (2.23.0)\n", | |
| "Collecting microconventions==0.4.6\n", | |
| " Downloading https://files.pythonhosted.org/packages/c0/26/3adcfa72642c14990f821e88ade830429bb5e61c9ae3f1bf3eb4b9c3068b/microconventions-0.4.6-py3-none-any.whl\n", | |
| "Requirement already satisfied: tzlocal>=1.2 in /usr/local/lib/python3.6/dist-packages (from apscheduler->microprediction) (1.5.1)\n", | |
| "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from apscheduler->microprediction) (1.15.0)\n", | |
| "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from apscheduler->microprediction) (2018.9)\n", | |
| "Requirement already satisfied: setuptools>=0.7 in /usr/local/lib/python3.6/dist-packages (from apscheduler->microprediction) (50.3.0)\n", | |
| "Collecting accumulation-tree\n", | |
| " Downloading https://files.pythonhosted.org/packages/72/82/6ca16f165b23f24d17303c6f7068be9519cdbe6cba5ad634160c5eeb53e8/accumulation_tree-0.6.2.tar.gz\n", | |
| "Collecting pyudorandom\n", | |
| " Downloading https://files.pythonhosted.org/packages/13/14/6fc20ea903eda547d6a255e995f8d4a09fdc3cf8bfacb6f85e6d669bc259/pyudorandom-1.0.0.tar.gz\n", | |
| "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt->microprediction) (0.16.0)\n", | |
| "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from hyperopt->microprediction) (3.11.0)\n", | |
| "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt->microprediction) (2.5)\n", | |
| "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from hyperopt->microprediction) (4.41.1)\n", | |
| "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from hyperopt->microprediction) (1.4.1)\n", | |
| "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->microprediction) (2.8.1)\n", | |
| "Collecting backoff\n", | |
| " Downloading https://files.pythonhosted.org/packages/f0/32/c5dd4f4b0746e9ec05ace2a5045c1fc375ae67ee94355344ad6c7005fd87/backoff-1.10.0-py2.py3-none-any.whl\n", | |
| "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->microprediction) (2.10)\n", | |
| "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->microprediction) (1.24.3)\n", | |
| "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->microprediction) (3.0.4)\n", | |
| "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->microprediction) (2020.6.20)\n", | |
| "Collecting pymorton\n", | |
| " Downloading https://files.pythonhosted.org/packages/c6/8d/906ba6d4266d7696547b8b70e08423975243c7339fe1ccf4bdbc42478394/pymorton-1.0.5-py2.py3-none-any.whl\n", | |
| "Collecting deepdiff\n", | |
| "\u001b[?25l Downloading https://files.pythonhosted.org/packages/67/26/49ccae53188f3bbe1fb6dd77721dd853b5eb5d72a932013931c429713839/deepdiff-5.0.2-py3-none-any.whl (60kB)\n", | |
| "\u001b[K |████████████████████████████████| 61kB 3.1MB/s \n", | |
| "\u001b[?25hCollecting muid>=0.5.0\n", | |
| "\u001b[?25l Downloading https://files.pythonhosted.org/packages/aa/29/a3d25b533a9b197e711d46a7e2d63497e435de900c293d7f0afbe37d121d/muid-0.5.0-py3-none-any.whl (173kB)\n", | |
| "\u001b[K |████████████████████████████████| 174kB 4.3MB/s \n", | |
| "\u001b[?25hCollecting schema\n", | |
| " Downloading https://files.pythonhosted.org/packages/6d/ae/835f2e0d304c9533c58fe5cbcdd9124708d32e82289fcb8d6084c908ba29/schema-0.7.2-py2.py3-none-any.whl\n", | |
| "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx->hyperopt->microprediction) (4.4.2)\n", | |
| "Collecting ordered-set>=4.0.1\n", | |
| " Downloading https://files.pythonhosted.org/packages/f5/ab/8252360bfe965bba31ec05112b3067bd129ce4800d89e0b85613bc6044f6/ordered-set-4.0.2.tar.gz\n", | |
| "Requirement already satisfied: contextlib2>=0.5.5 in /usr/local/lib/python3.6/dist-packages (from schema->microconventions==0.4.6->microprediction) (0.5.5)\n", | |
| "Building wheels for collected packages: contexttimer, genson, accumulation-tree, pyudorandom, ordered-set\n", | |
| " Building wheel for contexttimer (setup.py) ... \u001b[?25l\u001b[?25hdone\n", | |
| " Created wheel for contexttimer: filename=contexttimer-0.3.3-cp36-none-any.whl size=5819 sha256=56d00b5f330f4dc687de706067798f6a731c5055737470a090a293a11045696d\n", | |
| " Stored in directory: /root/.cache/pip/wheels/b3/e2/35/565145ce0127c7451b6503dfabb2b56e9908c863e40c6b1870\n", | |
| " Building wheel for genson (setup.py) ... \u001b[?25l\u001b[?25hdone\n", | |
| " Created wheel for genson: filename=genson-1.2.2-py2.py3-none-any.whl size=21261 sha256=cd1f35e55bc2bfdc701e0a627403d8cb05360fd5e4ff33241551bb48914d3c04\n", | |
| " Stored in directory: /root/.cache/pip/wheels/8e/49/09/a13c2bcc61a377c03552c85df8018dcc7706221913759b6707\n", | |
| " Building wheel for accumulation-tree (setup.py) ... \u001b[?25l\u001b[?25hdone\n", | |
| " Created wheel for accumulation-tree: filename=accumulation_tree-0.6.2-cp36-cp36m-linux_x86_64.whl size=237589 sha256=42ccc2a412f5fc933ff6ef98edb1c418c5c59d3ffaf75e88335f7ca385058baf\n", | |
| " Stored in directory: /root/.cache/pip/wheels/a7/76/fa/3fbcbcf87da8668138b93f9668f56e1029f7d72c7f6a51ab1a\n", | |
| " Building wheel for pyudorandom (setup.py) ... \u001b[?25l\u001b[?25hdone\n", | |
| " Created wheel for pyudorandom: filename=pyudorandom-1.0.0-cp36-none-any.whl size=2226 sha256=a45d13d1b46991ab47f467e2c3800abdb9a396056d74a438dc03cb3870731be7\n", | |
| " Stored in directory: /root/.cache/pip/wheels/18/0e/0c/ec5ba8bed98cea9fea1cee32ee83b57a5dfb2974b2777ac5a3\n", | |
| " Building wheel for ordered-set (setup.py) ... \u001b[?25l\u001b[?25hdone\n", | |
| " Created wheel for ordered-set: filename=ordered_set-4.0.2-py2.py3-none-any.whl size=8209 sha256=856cbf7fc2a5f3e9a7f175151ba88a554336249337fa8237f60a6fc8ba9548bb\n", | |
| " Stored in directory: /root/.cache/pip/wheels/e1/c6/9b/651d8a21d59b51a75ab9c070838f9231b8126421bc0569af47\n", | |
| "Successfully built contexttimer genson accumulation-tree pyudorandom ordered-set\n", | |
| "Installing collected packages: contexttimer, apscheduler, accumulation-tree, pyudorandom, tdigest, genson, pycoingecko, backoff, getjson, pymorton, ordered-set, deepdiff, muid, schema, microconventions, microprediction\n", | |
| "Successfully installed accumulation-tree-0.6.2 apscheduler-3.6.3 backoff-1.10.0 contexttimer-0.3.3 deepdiff-5.0.2 genson-1.2.2 getjson-1.0.0 microconventions-0.4.6 microprediction-0.15.4 muid-0.5.0 ordered-set-4.0.2 pycoingecko-1.4.0 pymorton-1.0.5 pyudorandom-1.0.0 schema-0.7.2 tdigest-0.5.2.2\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "jll4_B0yTNNX", | |
| "outputId": "c9b6cb04-97db-4b50-e287-9ced32eca7dd", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 35 | |
| } | |
| }, | |
| "source": [ | |
| "from microprediction import new_key\n", | |
| "write_key = new_key(difficulty=9)\n", | |
| "print(write_key)" | |
| ], | |
| "execution_count": 2, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "5546bd9f8944f3f3ee9d5ae6e9b613a9\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "jrFRZpZKTSsN" | |
| }, | |
| "source": [ | |
| "from microprediction import MicroCrawler\n", | |
| "\n", | |
| "class ElectricityCrawler(MicroCrawler):\n", | |
| "\n", | |
| " def __init__(self,**kwargs):\n", | |
| " super().__init__(**kwargs)\n", | |
| "\n", | |
| " def include_stream(self, name=None, **ignore):\n", | |
| " \"\"\" Override this as you see fit to select streams for your crawler\n", | |
| " For example your crawler might like z streams or not, or might require\n", | |
| " a minimum number of lags in the time series.\n", | |
| " \"\"\"\n", | |
| " return 'electricity' in name\n", | |
| "\n", | |
| " def exclude_stream(self, name=None, **ignore):\n", | |
| " \"\"\" Override this as you see fit to select streams for your crawler\n", | |
| " For example your crawler might like z streams or not, or might require\n", | |
| " a minimum number of lags in the time series.\n", | |
| " \"\"\"\n", | |
| " return '~' in name" | |
| ], | |
| "execution_count": 6, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "vZLWfI25T_51" | |
| }, | |
| "source": [ | |
| "crawler = ElectricityCrawler(write_key=\"5546bd9f8944f3f3ee9d5ae6e9b613a9\",min_len=100,stop_loss=3, max_active=100)\n" | |
| ], | |
| "execution_count": 10, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "OaE4dYAsUKG_", | |
| "outputId": "3580b6f7-da30-4456-fa26-357e8b1052de", | |
| "colab": { | |
| "base_uri": "https://localhost:8080/", | |
| "height": 783 | |
| } | |
| }, | |
| "source": [ | |
| "crawler.run()" | |
| ], | |
| "execution_count": null, | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "text": [ | |
| "Boost Mole restarting at 2020-10-13 23:38:51.509280\n", | |
| "{'current_balance': 0.0,\n", | |
| " 'currently_worst': [],\n", | |
| " 'min_lags': 25,\n", | |
| " 'num_active': 0,\n", | |
| " 'num_pending_cancellations': 0,\n", | |
| " 'pending_cancellations': [],\n", | |
| " 'quietude': 50,\n", | |
| " 'recent_errors': [],\n", | |
| " 'stop_loss': 3,\n", | |
| " 'upcoming': []}\n", | |
| "Boost Mole has active submissions for 0 horizons (0 with no withdrawal request pending since restart)\n", | |
| "Upcoming horizons and seconds to go ...\n", | |
| "[]\n", | |
| "{'current_balance': 0.0,\n", | |
| " 'currently_worst': [],\n", | |
| " 'min_lags': 25,\n", | |
| " 'num_active': 0,\n", | |
| " 'num_pending_cancellations': 0,\n", | |
| " 'pending_cancellations': [],\n", | |
| " 'quietude': 50,\n", | |
| " 'recent_errors': [],\n", | |
| " 'stop_loss': 3,\n", | |
| " 'upcoming': []}\n", | |
| "-------------------------------------------------------------\n", | |
| "Just a reminder ... \n", | |
| "Your write key is 5546bd9f8944f3f3ee9d5ae6e9b613a9\n", | |
| "Your public key is b0057301ebeb77d59dbb65ae98a60fa8\n", | |
| "Your nom de plume is Boost Mole\n", | |
| "Put your write key in the dashboard at www.microprediction.org\n", | |
| "--------------------------------------------------------------\n", | |
| "Checking performance \n", | |
| "Currently predicting for 0 horizons but found 102 candidate streams to examine.\n", | |
| "Submitted to electricity-lbmp-nyiso-o_h.json 910s horizon, and will do so again in 112 seconds.\n", | |
| "Submitted to horizon 910::electricity-lbmp-nyiso-o_h.json\n", | |
| "Currently predicting for 0 horizons but found 102 candidate streams to examine.\n", | |
| "Submitted to z1~electricity-lbmp-nyiso-longil~3555.json 3555s horizon, and will do so again in 5779 seconds.\n", | |
| "Submitted to horizon 3555::z1~electricity-lbmp-nyiso-longil~3555.json\n", | |
| "Downtime for 30s\n", | |
| "Boost Mole has active submissions for 0 horizons (0 with no withdrawal request pending since restart)\n", | |
| "Upcoming horizons and seconds to go ...\n", | |
| "[('910::electricity-lbmp-nyiso-o_h.json', 74.8),\n", | |
| " ('3555::z1~electricity-lbmp-nyiso-longil~3555.json', 5777.3)]\n" | |
| ], | |
| "name": "stdout" | |
| } | |
| ] | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
There was a mistake here. I accidentally referenced min_len in place of min_lags