Numerai Signals Google Trend.ipynb

numerai-signals-google-trend.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    },
    "colab": {
      "name": "Numerai Signals Google Trend.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/kunigaku/1395b89445f7c60f14b8de98402c1cb9/numerai-signals-google-trend.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dMUOW67hYb99"
      },
      "source": [
        "import datetime"
      ],
      "id": "dMUOW67hYb99",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "85ce78a5"
      },
      "source": [
        "today = datetime.datetime.now().strftime('%Y%m%d')\n",
        "hdf_name = f'trend_{today}.h5'\n",
        "hdf_name"
      ],
      "id": "85ce78a5",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0jjxF2s5IZ1J"
      },
      "source": [
        "!pip install pytrends numerapi"
      ],
      "id": "0jjxF2s5IZ1J",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U1hF76fLImmF"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd \"/content/drive/My Drive/signals\"\n",
        "import sys\n",
        "sys.path.append('/content/drive/My Drive/signals')"
      ],
      "id": "U1hF76fLImmF",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1600c21e"
      },
      "source": [
        "import numerapi\n",
        "import pandas as pd\n",
        "import time\n",
        "from pytrends.request import TrendReq"
      ],
      "id": "1600c21e",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36c73bca"
      },
      "source": [
        "napi = numerapi.SignalsAPI()"
      ],
      "id": "36c73bca",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1cf08049"
      },
      "source": [
        "eligible_tickers = pd.Series(napi.ticker_universe(), name='numerai_ticker')"
      ],
      "id": "1cf08049",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d28e41f3"
      },
      "source": [
        "ticker_map = pd.read_csv('https://numerai-signals-public-data.s3-us-west-2.amazonaws.com/signals_ticker_map_w_bbg.csv')"
      ],
      "id": "d28e41f3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "070c3407"
      },
      "source": [
        "trend_search_map = pd.read_csv('trend_search_list.csv')"
      ],
      "id": "070c3407",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ce5d1a45"
      },
      "source": [
        "trend_search_map"
      ],
      "id": "ce5d1a45",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0b54793a"
      },
      "source": [
        "ticker_map"
      ],
      "id": "0b54793a",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "873679c3"
      },
      "source": [
        "trend_search_map = pd.merge(ticker_map, trend_search_map, on='ticker')"
      ],
      "id": "873679c3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "acab739e"
      },
      "source": [
        "trend_search_map"
      ],
      "id": "acab739e",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1a181d46"
      },
      "source": [
        "trend_dfs = []"
      ],
      "id": "1a181d46",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aaeLKVMII9QM"
      },
      "source": [
        "for x in range(len(trend_search_map)):\n",
        "  try:\n",
        "    df = pd.read_hdf(hdf_name, f'trenddata_{x}')\n",
        "    trend_dfs.append(df)\n",
        "  except:\n",
        "    break"
      ],
      "id": "aaeLKVMII9QM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ZxO40tLbFOU"
      },
      "source": [
        "ticker2bloomberg = {}\n",
        "for x in ticker_map.itertuples():\n",
        "  # print(x.ticker, x.bloomberg_ticker)\n",
        "  ticker2bloomberg[x.ticker] = x.bloomberg_ticker"
      ],
      "id": "9ZxO40tLbFOU",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bj29BbLjGYbX"
      },
      "source": [
        "def save_trends():\n",
        "  for x in range(len(trend_dfs)):\n",
        "    df = trend_dfs[x]\n",
        "    print(x, df.columns[0])\n",
        "    df.to_hdf(hdf_name, f'trenddata_{x}')"
      ],
      "id": "bj29BbLjGYbX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17dc817f"
      },
      "source": [
        "pytrends = TrendReq(hl='en-US', tz=360, timeout=(10,25), retries=2, backoff_factor=0.1) # requests_args={'verify':False}\n",
        "# pytrends = TrendReq(hl='en-US', tz=360, timeout=(10,25), retries=10, backoff_factor=1) # requests_args={'verify':False}\n",
        "\n",
        "have_map = {}\n",
        "for df in trend_dfs:\n",
        "    have_map[df.columns[0]] = True\n",
        "\n",
        "for x in trend_search_map.itertuples():\n",
        "#     print(x)\n",
        "    if x.search_word in ['NASDAQ:CZR']:\n",
        "        print(f'skip {x.search_word}')\n",
        "        continue\n",
        "    if have_map.get(x.ticker) is None:\n",
        "        ok = False\n",
        "        print(x.search_word, x.trend_topics)\n",
        "#         print(x)\n",
        "        for trycount in range(5):\n",
        "          try:\n",
        "            kw_list = [x.trend_topics]\n",
        "            time.sleep(0.5)\n",
        "            pytrends.build_payload(kw_list=kw_list, timeframe='today 12-m', geo='', gprop='', cat=0)\n",
        "            z = pytrends.interest_over_time()\n",
        "            if len(z) > 0:\n",
        "    #             print(z)\n",
        "                z = z[x.trend_topics]\n",
        "                z = z.rename(x.ticker)\n",
        "                z = z.to_frame()\n",
        "                trend_dfs.append(z)\n",
        "            ok = True\n",
        "            break\n",
        "          except Exception as e:\n",
        "            print(e)\n",
        "        if not ok:\n",
        "          save_trends()\n",
        "          gotofail()\n",
        "#     break\n",
        "trend_dfs[-1]"
      ],
      "id": "17dc817f",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1dff1b36"
      },
      "source": [
        "save_trends()"
      ],
      "id": "1dff1b36",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f571c389"
      },
      "source": [
        "signals_dfs = []\n",
        "for df in trend_dfs:\n",
        "  ticker = df.columns[0]\n",
        "  print(ticker)\n",
        "  df = df.rename(columns={ticker: 'trend'})\n",
        "  df = df.replace(0, 1)\n",
        "  df['trend_ave13'] = df.rolling(13).mean()\n",
        "  df['trend_kairi13'] = (df['trend'] - df['trend_ave13']) / df['trend_ave13']\n",
        "  df['ticker'] = ticker\n",
        "  df['numerai_ticker'] = ticker2bloomberg[ticker]\n",
        "  df['friday_date'] = df.index + datetime.timedelta(days=5)\n",
        "  today = df.tail(1)['friday_date'][0]\n",
        "  df['data_type'] = df['friday_date'].apply(lambda x : 'live' if x == today else 'validation')\n",
        "  signals_dfs.append(df)\n",
        "# df"
      ],
      "id": "f571c389",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mKyzim5sdMc3"
      },
      "source": [
        "signal_df = pd.concat(signals_dfs)\n",
        "signal_df = signal_df.dropna()\n",
        "signal_df = signal_df.sort_index()\n",
        "signal_df"
      ],
      "id": "mKyzim5sdMc3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YuNgXHdmeQDe"
      },
      "source": [
        "signal_df = signal_df.reset_index()"
      ],
      "id": "YuNgXHdmeQDe",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C-IG86_befUV"
      },
      "source": [
        "def make_rank(pred):\n",
        "  return pred.rank(pct=True, method=\"first\")\n",
        "signal_df['signal'] = signal_df.groupby('date')['trend_kairi13'].apply(make_rank)\n",
        "signal_df"
      ],
      "id": "C-IG86_befUV",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YSjSj5LHgnDe"
      },
      "source": [
        "signal_df['friday_date'] = signal_df['friday_date'].dt.strftime('%Y%m%d')"
      ],
      "id": "YSjSj5LHgnDe",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uub0gnKIfelh"
      },
      "source": [
        "signal_df[['numerai_ticker', 'friday_date', 'data_type', 'signal']]"
      ],
      "id": "uub0gnKIfelh",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ITk4Ku9xgGZw"
      },
      "source": [
        "signal_df[['numerai_ticker', 'friday_date', 'data_type', 'signal']].to_csv('trend_signal_upload.csv', index=False)"
      ],
      "id": "ITk4Ku9xgGZw",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JNrVBxzBhUXr"
      },
      "source": [
        "public_id = \"YOUR_PUBLIC_ID\"\n",
        "secret_key = \"YOUR_SECRET_ID\"\n",
        "model_id = 'YOUR_MODEL_ID'\n",
        "napi = numerapi.SignalsAPI(public_id=public_id, secret_key=secret_key)\n",
        "napi.upload_predictions('trend_signal_upload.csv', model_id=model_id)"
      ],
      "id": "JNrVBxzBhUXr",
      "execution_count": null,
      "outputs": []
    }
  ]
}

trend_search_list.csv