Skip to content

Instantly share code, notes, and snippets.

@nagishin

nagishin/polars-pandas.ipynb

Last active February 17, 2023 03:52
Show Gist options
  • Save nagishin/858ebad76248cc129e7164e6a2d7d31a to your computer and use it in GitHub Desktop.
Save nagishin/858ebad76248cc129e7164e6a2d7d31a to your computer and use it in GitHub Desktop.
Download ZIP
polars-pandas.ipynb
Display the source blob
Display the rendered blob
Raw
polars-pandas.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/nagishin/858ebad76248cc129e7164e6a2d7d31a/polars-pandas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"source": [
"## 【polarsインストール】"
],
"metadata": {
"id": "SVmU4IjjAHYl"
}
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "LNKTrSWeVagD"
},
"outputs": [],
"source": [
"!pip install polars"
]
},
{
"cell_type": "markdown",
"source": [
"## 【初期設定】"
],
"metadata": {
"id": "gnaMWW_4AOgz"
}
},
{
"cell_type": "markdown",
"source": [
"### 約定履歴CSVの取得準備"
],
"metadata": {
"id": "rAE-p7IHMCNE"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"import time\n",
"from datetime import datetime\n",
"import pandas as pd\n",
"import polars as pl\n",
"\n",
"SYMBOL = 'BTCUSDT' # 通貨ペア\n",
"DATE = '2023-01-18' # 取得日\n",
"\n",
"DL_PATH = f'https://public.bybit.com/trading/{SYMBOL}/{SYMBOL}{DATE}.csv.gz' # ダウンロードパス\n",
"FILE_PATH = f'./trades_{SYMBOL}_{DATE}.csv' # CSVパス\n",
"\n",
"NANOSECONDS = 1_000_000_000 # nano秒単位\n",
"\n",
"# 処理結果DataFrame\n",
"result = pd.DataFrame({\n",
" '検証':['CSV読み込み', '約定履歴->OHLCV 1S', '約定履歴->OHLC 売買別出来高', 'OHLCV 1S->1H',],\n",
" 'pandas':[0.0, 0.0, 0.0, 0.0,],\n",
" 'polars':[0.0, 0.0, 0.0, 0.0,],\n",
"})\n",
"result.set_index('検証', drop=True, inplace=True)"
],
"metadata": {
"id": "wRpxWqtdWxYU"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"### 指定日の約定履歴をCSV保存"
],
"metadata": {
"id": "B9WRrTkiAUBE"
}
},
{
"cell_type": "code",
"source": [
"# ファイル存在チェック\n",
"if os.path.isfile(FILE_PATH) == False:\n",
" df_csv = pd.read_csv(DL_PATH, compression='gzip') # bybit約定履歴 1日分取得\n",
" df_csv.to_csv(FILE_PATH) # ローカルに保存\n",
"else:\n",
" df_csv = pd.read_csv(FILE_PATH) # ローカルCSV読み込み\n",
"\n",
"csv_records = len(df_csv)\n",
"csv_size = round(os.path.getsize(FILE_PATH)/1024/1024, 3)\n",
"\n",
"print(f'[約定履歴CSV]: {FILE_PATH}')\n",
"print(f'[レコード数 ]: {csv_records:,} rows')\n",
"print(f'[CSVサイズ ]: {csv_size:,} MB')\n",
"display(df_csv.head())"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 409
},
"id": "aQWfJRml4hC6",
"outputId": "cedb1b06-2ae2-4766-e250-60bd27ac607d"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[約定履歴CSV]: ./trades_BTCUSDT_2023-01-18.csv\n",
"[レコード数 ]: 1,386,138 rows\n",
"[CSVサイズ ]: 169.668 MB\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" Unnamed: 0 timestamp symbol side size price tickDirection \\\n",
"0 0 1.674000e+09 BTCUSDT Sell 0.010 21118.0 ZeroMinusTick \n",
"1 1 1.674000e+09 BTCUSDT Sell 0.024 21118.0 ZeroMinusTick \n",
"2 2 1.674000e+09 BTCUSDT Sell 0.063 21118.0 ZeroMinusTick \n",
"3 3 1.674000e+09 BTCUSDT Buy 0.001 21118.5 PlusTick \n",
"4 4 1.674000e+09 BTCUSDT Sell 0.337 21118.0 MinusTick \n",
"\n",
" trdMatchID grossValue homeNotional \\\n",
"0 9165901d-8b04-5bdb-bba5-7365be082394 2.111800e+10 0.010 \n",
"1 85505b5d-6386-5054-a61d-f332e83ba3e6 5.068320e+10 0.024 \n",
"2 749209ec-92a7-5b8d-81d8-1dd0fdc9a6f4 1.330434e+11 0.063 \n",
"3 98f7e7ec-a266-5909-868c-1677554a79f9 2.111850e+09 0.001 \n",
"4 ba378ff8-3c09-5294-a2d4-a4c9defad9aa 7.116766e+11 0.337 \n",
"\n",
" foreignNotional \n",
"0 211.1800 \n",
"1 506.8320 \n",
"2 1330.4340 \n",
"3 21.1185 \n",
"4 7116.7660 "
],
"text/html": [
"\n",
" <div id=\"df-68a6be58-7187-47f7-ad3f-ab211ca2983e\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Unnamed: 0</th>\n",
" <th>timestamp</th>\n",
" <th>symbol</th>\n",
" <th>side</th>\n",
" <th>size</th>\n",
" <th>price</th>\n",
" <th>tickDirection</th>\n",
" <th>trdMatchID</th>\n",
" <th>grossValue</th>\n",
" <th>homeNotional</th>\n",
" <th>foreignNotional</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>1.674000e+09</td>\n",
" <td>BTCUSDT</td>\n",
" <td>Sell</td>\n",
" <td>0.010</td>\n",
" <td>21118.0</td>\n",
" <td>ZeroMinusTick</td>\n",
" <td>9165901d-8b04-5bdb-bba5-7365be082394</td>\n",
" <td>2.111800e+10</td>\n",
" <td>0.010</td>\n",
" <td>211.1800</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>1.674000e+09</td>\n",
" <td>BTCUSDT</td>\n",
" <td>Sell</td>\n",
" <td>0.024</td>\n",
" <td>21118.0</td>\n",
" <td>ZeroMinusTick</td>\n",
" <td>85505b5d-6386-5054-a61d-f332e83ba3e6</td>\n",
" <td>5.068320e+10</td>\n",
" <td>0.024</td>\n",
" <td>506.8320</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>1.674000e+09</td>\n",
" <td>BTCUSDT</td>\n",
" <td>Sell</td>\n",
" <td>0.063</td>\n",
" <td>21118.0</td>\n",
" <td>ZeroMinusTick</td>\n",
" <td>749209ec-92a7-5b8d-81d8-1dd0fdc9a6f4</td>\n",
" <td>1.330434e+11</td>\n",
" <td>0.063</td>\n",
" <td>1330.4340</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>1.674000e+09</td>\n",
" <td>BTCUSDT</td>\n",
" <td>Buy</td>\n",
" <td>0.001</td>\n",
" <td>21118.5</td>\n",
" <td>PlusTick</td>\n",
" <td>98f7e7ec-a266-5909-868c-1677554a79f9</td>\n",
" <td>2.111850e+09</td>\n",
" <td>0.001</td>\n",
" <td>21.1185</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>1.674000e+09</td>\n",
" <td>BTCUSDT</td>\n",
" <td>Sell</td>\n",
" <td>0.337</td>\n",
" <td>21118.0</td>\n",
" <td>MinusTick</td>\n",
" <td>ba378ff8-3c09-5294-a2d4-a4c9defad9aa</td>\n",
" <td>7.116766e+11</td>\n",
" <td>0.337</td>\n",
" <td>7116.7660</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-68a6be58-7187-47f7-ad3f-ab211ca2983e')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-68a6be58-7187-47f7-ad3f-ab211ca2983e button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-68a6be58-7187-47f7-ad3f-ab211ca2983e');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"## 【1. CSV読み込み比較】\n",
"* 約定履歴CSVファイルから列選択して読み込み\n",
"* UnixTime->Datetime列追加"
],
"metadata": {
"id": "FKp9Axw5AlL7"
}
},
{
"cell_type": "markdown",
"source": [
"### pandas"
],
"metadata": {
"id": "A8x0wi7iAx38"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 約定履歴csvをDataFrameに読み込み\n",
"df_trade_pd = pd.read_csv(FILE_PATH,\n",
" usecols=['timestamp', 'side', 'price', 'size'],\n",
" dtype={'timestamp':'float', 'side':'str', 'price':'float', 'size':'float'})\n",
"\n",
"# UnixTime(timestamp)からdatetime列生成\n",
"df_trade_pd['datetime'] = pd.to_datetime(df_trade_pd['timestamp'], unit='s', utc=True)\n",
"\n",
"# datetimeをindexに設定 (resampleのため)\n",
"df_trade_pd.set_index('datetime', inplace=True)\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['CSV読み込み']['pandas'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_trade_pd.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 274
},
"id": "mn0_JJYxV4qa",
"outputId": "0c52a9b4-8577-4559-cdf7-09882042a73f"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 1997.358 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" timestamp side size price\n",
"datetime \n",
"2023-01-18 00:00:01.136000+00:00 1.674000e+09 Sell 0.010 21118.0\n",
"2023-01-18 00:00:01.379800064+00:00 1.674000e+09 Sell 0.024 21118.0\n",
"2023-01-18 00:00:01.530400+00:00 1.674000e+09 Sell 0.063 21118.0\n",
"2023-01-18 00:00:01.541400064+00:00 1.674000e+09 Buy 0.001 21118.5\n",
"2023-01-18 00:00:01.543100160+00:00 1.674000e+09 Sell 0.337 21118.0"
],
"text/html": [
"\n",
" <div id=\"df-1b1bdd17-eb40-427b-ab77-9beb35e13660\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>timestamp</th>\n",
" <th>side</th>\n",
" <th>size</th>\n",
" <th>price</th>\n",
" </tr>\n",
" <tr>\n",
" <th>datetime</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2023-01-18 00:00:01.136000+00:00</th>\n",
" <td>1.674000e+09</td>\n",
" <td>Sell</td>\n",
" <td>0.010</td>\n",
" <td>21118.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:01.379800064+00:00</th>\n",
" <td>1.674000e+09</td>\n",
" <td>Sell</td>\n",
" <td>0.024</td>\n",
" <td>21118.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:01.530400+00:00</th>\n",
" <td>1.674000e+09</td>\n",
" <td>Sell</td>\n",
" <td>0.063</td>\n",
" <td>21118.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:01.541400064+00:00</th>\n",
" <td>1.674000e+09</td>\n",
" <td>Buy</td>\n",
" <td>0.001</td>\n",
" <td>21118.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:01.543100160+00:00</th>\n",
" <td>1.674000e+09</td>\n",
" <td>Sell</td>\n",
" <td>0.337</td>\n",
" <td>21118.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1b1bdd17-eb40-427b-ab77-9beb35e13660')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-1b1bdd17-eb40-427b-ab77-9beb35e13660 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-1b1bdd17-eb40-427b-ab77-9beb35e13660');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"### polars"
],
"metadata": {
"id": "_3gQqDUkAq-r"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 約定履歴csvをDataFrameに読み込み\n",
"df_trade_pl = (\n",
" pl.read_csv(FILE_PATH, # CSVパス\n",
" columns=['timestamp', 'side', 'price', 'size'], # 列選択\n",
" dtypes={'timestamp':pl.Float64, 'side':pl.Utf8, 'price':pl.Float64, 'size':pl.Float64}) # 列型指定\n",
" .with_columns( # 列追加\n",
" (pl.col('timestamp') * NANOSECONDS) # timestamp(秒)をnano秒単位に\n",
" .cast(pl.Datetime(time_unit='ns')) # datetime(nano秒)変換\n",
" .dt.replace_time_zone('UTC') # タイムゾーン設定\n",
" .alias('datetime') # 列名設定\n",
" )\n",
")\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['CSV読み込み']['polars'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_trade_pl.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 267
},
"id": "__ckOQvKWp8b",
"outputId": "caa4950f-f448-48b7-f007-6c31b72b3776"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 917.156 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"shape: (5, 5)\n",
"┌───────────┬──────┬───────┬─────────┬─────────────────────────────────────┐\n",
"│ timestamp ┆ side ┆ size ┆ price ┆ datetime │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ f64 ┆ str ┆ f64 ┆ f64 ┆ datetime[ns, UTC] │\n",
"╞═══════════╪══════╪═══════╪═════════╪═════════════════════════════════════╡\n",
"│ 1.6740e9 ┆ Sell ┆ 0.01 ┆ 21118.0 ┆ 2023-01-18 00:00:01.136 UTC │\n",
"│ 1.6740e9 ┆ Sell ┆ 0.024 ┆ 21118.0 ┆ 2023-01-18 00:00:01.379800064 UT... │\n",
"│ 1.6740e9 ┆ Sell ┆ 0.063 ┆ 21118.0 ┆ 2023-01-18 00:00:01.530400 UTC │\n",
"│ 1.6740e9 ┆ Buy ┆ 0.001 ┆ 21118.5 ┆ 2023-01-18 00:00:01.541400064 UT... │\n",
"│ 1.6740e9 ┆ Sell ┆ 0.337 ┆ 21118.0 ┆ 2023-01-18 00:00:01.543100160 UT... │\n",
"└───────────┴──────┴───────┴─────────┴─────────────────────────────────────┘"
],
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe td {\n",
" white-space: pre;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-top: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-bottom: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" line-height: 95%;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
"<small>shape: (5, 5)</small>\n",
"<thead>\n",
"<tr>\n",
"<th>\n",
"timestamp\n",
"</th>\n",
"<th>\n",
"side\n",
"</th>\n",
"<th>\n",
"size\n",
"</th>\n",
"<th>\n",
"price\n",
"</th>\n",
"<th>\n",
"datetime\n",
"</th>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"str\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"datetime[ns, UTC]\n",
"</td>\n",
"</tr>\n",
"</thead>\n",
"<tbody>\n",
"<tr>\n",
"<td>\n",
"1.6740e9\n",
"</td>\n",
"<td>\n",
"&quot;Sell&quot;\n",
"</td>\n",
"<td>\n",
"0.01\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"2023-01-18 00:00:01.136 UTC\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"1.6740e9\n",
"</td>\n",
"<td>\n",
"&quot;Sell&quot;\n",
"</td>\n",
"<td>\n",
"0.024\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"2023-01-18 00:00:01.379800064 UTC\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"1.6740e9\n",
"</td>\n",
"<td>\n",
"&quot;Sell&quot;\n",
"</td>\n",
"<td>\n",
"0.063\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"2023-01-18 00:00:01.530400 UTC\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"1.6740e9\n",
"</td>\n",
"<td>\n",
"&quot;Buy&quot;\n",
"</td>\n",
"<td>\n",
"0.001\n",
"</td>\n",
"<td>\n",
"21118.5\n",
"</td>\n",
"<td>\n",
"2023-01-18 00:00:01.541400064 UTC\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"1.6740e9\n",
"</td>\n",
"<td>\n",
"&quot;Sell&quot;\n",
"</td>\n",
"<td>\n",
"0.337\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"2023-01-18 00:00:01.543100160 UTC\n",
"</td>\n",
"</tr>\n",
"</tbody>\n",
"</table>\n",
"</div>"
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"## 【2. 約定履歴->OHLCV(1秒足)】"
],
"metadata": {
"id": "aae9xeJoA2PM"
}
},
{
"cell_type": "markdown",
"source": [
"### pandas"
],
"metadata": {
"id": "8UBeAI6KBBQM"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 約定履歴 -> OHLCV\n",
"df_ohlcv_pd = df_trade_pd.resample('1S').agg({\n",
" 'price' : 'ohlc',\n",
" 'size' : 'sum',}).ffill()\n",
"# 列名編集\n",
"df_ohlcv_pd.columns = ['open', 'high', 'low', 'close', 'volume']\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['約定履歴->OHLCV 1S']['pandas'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_ohlcv_pd.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 274
},
"id": "gErkgKVGbopj",
"outputId": "e5d2c502-01ed-4385-e5ee-10c4863051d8"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 115.377 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" open high low close volume\n",
"datetime \n",
"2023-01-18 00:00:01+00:00 21118.0 21118.5 21118.0 21118.0 0.464\n",
"2023-01-18 00:00:02+00:00 21118.0 21118.0 21118.0 21118.0 0.211\n",
"2023-01-18 00:00:03+00:00 21118.0 21118.0 21118.0 21118.0 0.011\n",
"2023-01-18 00:00:04+00:00 21118.0 21118.5 21118.0 21118.5 0.014\n",
"2023-01-18 00:00:05+00:00 21118.0 21118.5 21118.0 21118.0 1.118"
],
"text/html": [
"\n",
" <div id=\"df-d673db9d-857f-4414-8f8d-6bf5d030f97e\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>open</th>\n",
" <th>high</th>\n",
" <th>low</th>\n",
" <th>close</th>\n",
" <th>volume</th>\n",
" </tr>\n",
" <tr>\n",
" <th>datetime</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2023-01-18 00:00:01+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21118.5</td>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>0.464</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:02+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>0.211</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:03+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>0.011</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:04+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21118.5</td>\n",
" <td>21118.0</td>\n",
" <td>21118.5</td>\n",
" <td>0.014</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:00:05+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21118.5</td>\n",
" <td>21118.0</td>\n",
" <td>21118.0</td>\n",
" <td>1.118</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d673db9d-857f-4414-8f8d-6bf5d030f97e')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-d673db9d-857f-4414-8f8d-6bf5d030f97e button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-d673db9d-857f-4414-8f8d-6bf5d030f97e');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"### polars"
],
"metadata": {
"id": "KAvHBSokA-50"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 約定履歴 -> OHLCV\n",
"df_ohlcv_pl = (\n",
" df_trade_pl.groupby_dynamic('datetime', every='1s') # datetimeを1秒毎にグループ化\n",
" .agg([\n",
" pl.col('price').first().alias('open'),\n",
" pl.col('price').max().alias('high'),\n",
" pl.col('price').min().alias('low'),\n",
" pl.col('price').last().alias('close'),\n",
" pl.col('size').sum().alias('volume')\n",
" ])\n",
")\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['約定履歴->OHLCV 1S']['polars'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_ohlcv_pl.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 267
},
"id": "yW6r0zcNoZoO",
"outputId": "078f28ce-f7a2-468e-bb6f-9a2f98495a88"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 44.667 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"shape: (5, 6)\n",
"┌─────────────────────────┬─────────┬─────────┬─────────┬─────────┬────────┐\n",
"│ datetime ┆ open ┆ high ┆ low ┆ close ┆ volume │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ datetime[ns, UTC] ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n",
"╞═════════════════════════╪═════════╪═════════╪═════════╪═════════╪════════╡\n",
"│ 2023-01-18 00:00:01 UTC ┆ 21118.0 ┆ 21118.5 ┆ 21118.0 ┆ 21118.0 ┆ 0.464 │\n",
"│ 2023-01-18 00:00:02 UTC ┆ 21118.0 ┆ 21118.0 ┆ 21118.0 ┆ 21118.0 ┆ 0.211 │\n",
"│ 2023-01-18 00:00:03 UTC ┆ 21118.0 ┆ 21118.0 ┆ 21118.0 ┆ 21118.0 ┆ 0.011 │\n",
"│ 2023-01-18 00:00:04 UTC ┆ 21118.0 ┆ 21118.5 ┆ 21118.0 ┆ 21118.5 ┆ 0.014 │\n",
"│ 2023-01-18 00:00:05 UTC ┆ 21118.0 ┆ 21118.5 ┆ 21118.0 ┆ 21118.0 ┆ 1.118 │\n",
"└─────────────────────────┴─────────┴─────────┴─────────┴─────────┴────────┘"
],
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe td {\n",
" white-space: pre;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-top: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-bottom: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" line-height: 95%;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
"<small>shape: (5, 6)</small>\n",
"<thead>\n",
"<tr>\n",
"<th>\n",
"datetime\n",
"</th>\n",
"<th>\n",
"open\n",
"</th>\n",
"<th>\n",
"high\n",
"</th>\n",
"<th>\n",
"low\n",
"</th>\n",
"<th>\n",
"close\n",
"</th>\n",
"<th>\n",
"volume\n",
"</th>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"datetime[ns, UTC]\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"</tr>\n",
"</thead>\n",
"<tbody>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:01 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.5\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"0.464\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:02 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"0.211\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:03 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"0.011\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:04 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.5\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.5\n",
"</td>\n",
"<td>\n",
"0.014\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:05 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.5\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"1.118\n",
"</td>\n",
"</tr>\n",
"</tbody>\n",
"</table>\n",
"</div>"
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"## 【3. 約定履歴->OHLCV(1分足 売買別出来高集計)】"
],
"metadata": {
"id": "MB-QsPgICDN8"
}
},
{
"cell_type": "markdown",
"source": [
"### pandas"
],
"metadata": {
"id": "NCjmqXbACeP8"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 約定サイズを売買分類\n",
"df_trade_pd['buy_size'] = df_trade_pd['size'].where(df_trade_pd['side'] == 'Buy', 0)\n",
"df_trade_pd['sell_size'] = df_trade_pd['size'].where(df_trade_pd['side'] == 'Sell', 0)\n",
"\n",
"# 約定履歴 -> OHLCV(売買別volume)\n",
"df_ohlcv_pd2 = df_trade_pd.resample('1T').agg({\n",
" 'price' : 'ohlc',\n",
" 'buy_size' : 'sum',\n",
" 'sell_size' : 'sum',}).ffill()\n",
"# 列名編集\n",
"df_ohlcv_pd2.columns = ['open', 'high', 'low', 'close', 'buy_vol', 'sell_vol']\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['約定履歴->OHLC 売買別出来高']['pandas'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_ohlcv_pd2.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 274
},
"id": "EUko_R0hBtoZ",
"outputId": "4eeba50e-0b81-43b6-ccee-4e5691dab97c"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 443.944 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" open high low close buy_vol \\\n",
"datetime \n",
"2023-01-18 00:00:00+00:00 21118.0 21130.5 21113.0 21130.0 175.302 \n",
"2023-01-18 00:01:00+00:00 21130.0 21137.0 21130.0 21135.5 82.374 \n",
"2023-01-18 00:02:00+00:00 21135.0 21144.0 21131.0 21140.5 82.671 \n",
"2023-01-18 00:03:00+00:00 21140.5 21155.0 21140.5 21155.0 118.928 \n",
"2023-01-18 00:04:00+00:00 21154.5 21155.0 21148.0 21148.5 59.904 \n",
"\n",
" sell_vol \n",
"datetime \n",
"2023-01-18 00:00:00+00:00 183.885 \n",
"2023-01-18 00:01:00+00:00 17.025 \n",
"2023-01-18 00:02:00+00:00 16.217 \n",
"2023-01-18 00:03:00+00:00 73.689 \n",
"2023-01-18 00:04:00+00:00 20.405 "
],
"text/html": [
"\n",
" <div id=\"df-d595c80f-bc7a-459d-a9c2-49f50b867585\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>open</th>\n",
" <th>high</th>\n",
" <th>low</th>\n",
" <th>close</th>\n",
" <th>buy_vol</th>\n",
" <th>sell_vol</th>\n",
" </tr>\n",
" <tr>\n",
" <th>datetime</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2023-01-18 00:00:00+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21130.5</td>\n",
" <td>21113.0</td>\n",
" <td>21130.0</td>\n",
" <td>175.302</td>\n",
" <td>183.885</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:01:00+00:00</th>\n",
" <td>21130.0</td>\n",
" <td>21137.0</td>\n",
" <td>21130.0</td>\n",
" <td>21135.5</td>\n",
" <td>82.374</td>\n",
" <td>17.025</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:02:00+00:00</th>\n",
" <td>21135.0</td>\n",
" <td>21144.0</td>\n",
" <td>21131.0</td>\n",
" <td>21140.5</td>\n",
" <td>82.671</td>\n",
" <td>16.217</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:03:00+00:00</th>\n",
" <td>21140.5</td>\n",
" <td>21155.0</td>\n",
" <td>21140.5</td>\n",
" <td>21155.0</td>\n",
" <td>118.928</td>\n",
" <td>73.689</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 00:04:00+00:00</th>\n",
" <td>21154.5</td>\n",
" <td>21155.0</td>\n",
" <td>21148.0</td>\n",
" <td>21148.5</td>\n",
" <td>59.904</td>\n",
" <td>20.405</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d595c80f-bc7a-459d-a9c2-49f50b867585')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-d595c80f-bc7a-459d-a9c2-49f50b867585 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-d595c80f-bc7a-459d-a9c2-49f50b867585');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"### polars"
],
"metadata": {
"id": "fQDUZ9_kHG8l"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 約定履歴 -> OHLCV(売買別volume)\n",
"df_ohlcv_pl2 = (\n",
" df_trade_pl\n",
" .with_columns(pl.when(pl.col('side') == 'Buy').then(pl.col('size')).otherwise(0).alias('buy_size')) # Buyのみのsize列追加\n",
" .with_columns(pl.when(pl.col('side') == 'Sell').then(pl.col('size')).otherwise(0).alias('sell_size')) # Sellのみのsize列追加\n",
" .groupby_dynamic('datetime', every='1m') # datetimeを1分毎にグループ化\n",
" .agg([\n",
" pl.col('price').first().alias('open'),\n",
" pl.col('price').max().alias('high'),\n",
" pl.col('price').min().alias('low'),\n",
" pl.col('price').last().alias('close'),\n",
" pl.col('buy_size').sum().alias('buy_vol'),\n",
" pl.col('sell_size').sum().alias('sell_vol'),\n",
" ])\n",
")\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['約定履歴->OHLC 売買別出来高']['polars'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_ohlcv_pl2.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 267
},
"id": "7eFNnrnfCg6E",
"outputId": "57ae5d43-09f7-4d44-d0b0-d11356a9c801"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 87.974 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"shape: (5, 7)\n",
"┌─────────────────────────┬─────────┬─────────┬─────────┬─────────┬─────────┬──────────┐\n",
"│ datetime ┆ open ┆ high ┆ low ┆ close ┆ buy_vol ┆ sell_vol │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ datetime[ns, UTC] ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n",
"╞═════════════════════════╪═════════╪═════════╪═════════╪═════════╪═════════╪══════════╡\n",
"│ 2023-01-18 00:00:00 UTC ┆ 21118.0 ┆ 21130.5 ┆ 21113.0 ┆ 21130.0 ┆ 175.302 ┆ 183.885 │\n",
"│ 2023-01-18 00:01:00 UTC ┆ 21130.0 ┆ 21137.0 ┆ 21130.0 ┆ 21135.5 ┆ 82.374 ┆ 17.025 │\n",
"│ 2023-01-18 00:02:00 UTC ┆ 21135.0 ┆ 21144.0 ┆ 21131.0 ┆ 21140.5 ┆ 82.671 ┆ 16.217 │\n",
"│ 2023-01-18 00:03:00 UTC ┆ 21140.5 ┆ 21155.0 ┆ 21140.5 ┆ 21155.0 ┆ 118.928 ┆ 73.689 │\n",
"│ 2023-01-18 00:04:00 UTC ┆ 21154.5 ┆ 21155.0 ┆ 21148.0 ┆ 21148.5 ┆ 59.904 ┆ 20.405 │\n",
"└─────────────────────────┴─────────┴─────────┴─────────┴─────────┴─────────┴──────────┘"
],
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe td {\n",
" white-space: pre;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-top: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-bottom: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" line-height: 95%;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
"<small>shape: (5, 7)</small>\n",
"<thead>\n",
"<tr>\n",
"<th>\n",
"datetime\n",
"</th>\n",
"<th>\n",
"open\n",
"</th>\n",
"<th>\n",
"high\n",
"</th>\n",
"<th>\n",
"low\n",
"</th>\n",
"<th>\n",
"close\n",
"</th>\n",
"<th>\n",
"buy_vol\n",
"</th>\n",
"<th>\n",
"sell_vol\n",
"</th>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"datetime[ns, UTC]\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"</tr>\n",
"</thead>\n",
"<tbody>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:00 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21130.5\n",
"</td>\n",
"<td>\n",
"21113.0\n",
"</td>\n",
"<td>\n",
"21130.0\n",
"</td>\n",
"<td>\n",
"175.302\n",
"</td>\n",
"<td>\n",
"183.885\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:01:00 UTC\n",
"</td>\n",
"<td>\n",
"21130.0\n",
"</td>\n",
"<td>\n",
"21137.0\n",
"</td>\n",
"<td>\n",
"21130.0\n",
"</td>\n",
"<td>\n",
"21135.5\n",
"</td>\n",
"<td>\n",
"82.374\n",
"</td>\n",
"<td>\n",
"17.025\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:02:00 UTC\n",
"</td>\n",
"<td>\n",
"21135.0\n",
"</td>\n",
"<td>\n",
"21144.0\n",
"</td>\n",
"<td>\n",
"21131.0\n",
"</td>\n",
"<td>\n",
"21140.5\n",
"</td>\n",
"<td>\n",
"82.671\n",
"</td>\n",
"<td>\n",
"16.217\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:03:00 UTC\n",
"</td>\n",
"<td>\n",
"21140.5\n",
"</td>\n",
"<td>\n",
"21155.0\n",
"</td>\n",
"<td>\n",
"21140.5\n",
"</td>\n",
"<td>\n",
"21155.0\n",
"</td>\n",
"<td>\n",
"118.928\n",
"</td>\n",
"<td>\n",
"73.689\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:04:00 UTC\n",
"</td>\n",
"<td>\n",
"21154.5\n",
"</td>\n",
"<td>\n",
"21155.0\n",
"</td>\n",
"<td>\n",
"21148.0\n",
"</td>\n",
"<td>\n",
"21148.5\n",
"</td>\n",
"<td>\n",
"59.904\n",
"</td>\n",
"<td>\n",
"20.405\n",
"</td>\n",
"</tr>\n",
"</tbody>\n",
"</table>\n",
"</div>"
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"## 【4. ダウンサンプリング(1秒足->1時間足)】"
],
"metadata": {
"id": "Weo9ItPVBPxE"
}
},
{
"cell_type": "markdown",
"source": [
"### pandas"
],
"metadata": {
"id": "wGAmuk37hZvQ"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"df_ohlcv_1h_pd = df_ohlcv_pd.resample('1H').agg({\n",
" 'open' : 'first',\n",
" 'high' : 'max',\n",
" 'low' : 'min',\n",
" 'close' : 'last',\n",
" 'volume' : 'sum',})\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['OHLCV 1S->1H']['pandas'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_ohlcv_1h_pd.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 274
},
"id": "gUuS28VKINMn",
"outputId": "185cd4d6-34fc-4db9-9e60-51ce0c377ca3"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 13.292 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" open high low close volume\n",
"datetime \n",
"2023-01-18 00:00:00+00:00 21118.0 21231.0 21086.0 21200.5 5330.769\n",
"2023-01-18 01:00:00+00:00 21200.5 21261.0 21175.0 21239.0 2704.494\n",
"2023-01-18 02:00:00+00:00 21238.5 21355.5 21143.0 21285.0 5764.510\n",
"2023-01-18 03:00:00+00:00 21285.5 21369.5 21255.0 21255.5 5015.602\n",
"2023-01-18 04:00:00+00:00 21255.0 21291.0 21220.5 21272.5 2342.050"
],
"text/html": [
"\n",
" <div id=\"df-e3562e24-fb0b-4094-bcef-3503700864ce\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>open</th>\n",
" <th>high</th>\n",
" <th>low</th>\n",
" <th>close</th>\n",
" <th>volume</th>\n",
" </tr>\n",
" <tr>\n",
" <th>datetime</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2023-01-18 00:00:00+00:00</th>\n",
" <td>21118.0</td>\n",
" <td>21231.0</td>\n",
" <td>21086.0</td>\n",
" <td>21200.5</td>\n",
" <td>5330.769</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 01:00:00+00:00</th>\n",
" <td>21200.5</td>\n",
" <td>21261.0</td>\n",
" <td>21175.0</td>\n",
" <td>21239.0</td>\n",
" <td>2704.494</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 02:00:00+00:00</th>\n",
" <td>21238.5</td>\n",
" <td>21355.5</td>\n",
" <td>21143.0</td>\n",
" <td>21285.0</td>\n",
" <td>5764.510</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 03:00:00+00:00</th>\n",
" <td>21285.5</td>\n",
" <td>21369.5</td>\n",
" <td>21255.0</td>\n",
" <td>21255.5</td>\n",
" <td>5015.602</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2023-01-18 04:00:00+00:00</th>\n",
" <td>21255.0</td>\n",
" <td>21291.0</td>\n",
" <td>21220.5</td>\n",
" <td>21272.5</td>\n",
" <td>2342.050</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e3562e24-fb0b-4094-bcef-3503700864ce')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-e3562e24-fb0b-4094-bcef-3503700864ce button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-e3562e24-fb0b-4094-bcef-3503700864ce');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"### polars"
],
"metadata": {
"id": "wNoTLdzihWra"
}
},
{
"cell_type": "code",
"source": [
"s = time.time()\n",
"\n",
"# 1秒足 -> 1時間足\n",
"df_ohlcv_1h_pl = (\n",
" df_ohlcv_pl.groupby_dynamic('datetime', every='1h') # datetimeを1時間毎にグループ化\n",
" .agg([\n",
" pl.col('open').first(),\n",
" pl.col('high').max(),\n",
" pl.col('low').min(),\n",
" pl.col('close').last(),\n",
" pl.col('volume').sum()\n",
" ])\n",
")\n",
"\n",
"ret = round((time.time() - s) * 1000, 3)\n",
"result.loc['OHLCV 1S->1H']['polars'] = ret\n",
"\n",
"print(f'[処理時間]: {ret} ms\\n')\n",
"display(df_ohlcv_1h_pl.head(5))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 267
},
"id": "a82y_UYsoGr3",
"outputId": "5b9ee3ec-31f1-4b6e-9907-15a5f3276a77"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[処理時間]: 2.619 ms\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"shape: (5, 6)\n",
"┌─────────────────────────┬─────────┬─────────┬─────────┬─────────┬──────────┐\n",
"│ datetime ┆ open ┆ high ┆ low ┆ close ┆ volume │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ datetime[ns, UTC] ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n",
"╞═════════════════════════╪═════════╪═════════╪═════════╪═════════╪══════════╡\n",
"│ 2023-01-18 00:00:00 UTC ┆ 21118.0 ┆ 21231.0 ┆ 21086.0 ┆ 21200.5 ┆ 5330.769 │\n",
"│ 2023-01-18 01:00:00 UTC ┆ 21200.5 ┆ 21261.0 ┆ 21175.0 ┆ 21239.0 ┆ 2704.494 │\n",
"│ 2023-01-18 02:00:00 UTC ┆ 21238.5 ┆ 21355.5 ┆ 21143.0 ┆ 21285.0 ┆ 5764.51 │\n",
"│ 2023-01-18 03:00:00 UTC ┆ 21285.5 ┆ 21369.5 ┆ 21255.0 ┆ 21255.5 ┆ 5015.602 │\n",
"│ 2023-01-18 04:00:00 UTC ┆ 21255.0 ┆ 21291.0 ┆ 21220.5 ┆ 21272.5 ┆ 2342.05 │\n",
"└─────────────────────────┴─────────┴─────────┴─────────┴─────────┴──────────┘"
],
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe td {\n",
" white-space: pre;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-top: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" padding-bottom: 0;\n",
" }\n",
"\n",
" .dataframe td {\n",
" line-height: 95%;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
"<small>shape: (5, 6)</small>\n",
"<thead>\n",
"<tr>\n",
"<th>\n",
"datetime\n",
"</th>\n",
"<th>\n",
"open\n",
"</th>\n",
"<th>\n",
"high\n",
"</th>\n",
"<th>\n",
"low\n",
"</th>\n",
"<th>\n",
"close\n",
"</th>\n",
"<th>\n",
"volume\n",
"</th>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"datetime[ns, UTC]\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"<td>\n",
"f64\n",
"</td>\n",
"</tr>\n",
"</thead>\n",
"<tbody>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 00:00:00 UTC\n",
"</td>\n",
"<td>\n",
"21118.0\n",
"</td>\n",
"<td>\n",
"21231.0\n",
"</td>\n",
"<td>\n",
"21086.0\n",
"</td>\n",
"<td>\n",
"21200.5\n",
"</td>\n",
"<td>\n",
"5330.769\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 01:00:00 UTC\n",
"</td>\n",
"<td>\n",
"21200.5\n",
"</td>\n",
"<td>\n",
"21261.0\n",
"</td>\n",
"<td>\n",
"21175.0\n",
"</td>\n",
"<td>\n",
"21239.0\n",
"</td>\n",
"<td>\n",
"2704.494\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 02:00:00 UTC\n",
"</td>\n",
"<td>\n",
"21238.5\n",
"</td>\n",
"<td>\n",
"21355.5\n",
"</td>\n",
"<td>\n",
"21143.0\n",
"</td>\n",
"<td>\n",
"21285.0\n",
"</td>\n",
"<td>\n",
"5764.51\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 03:00:00 UTC\n",
"</td>\n",
"<td>\n",
"21285.5\n",
"</td>\n",
"<td>\n",
"21369.5\n",
"</td>\n",
"<td>\n",
"21255.0\n",
"</td>\n",
"<td>\n",
"21255.5\n",
"</td>\n",
"<td>\n",
"5015.602\n",
"</td>\n",
"</tr>\n",
"<tr>\n",
"<td>\n",
"2023-01-18 04:00:00 UTC\n",
"</td>\n",
"<td>\n",
"21255.0\n",
"</td>\n",
"<td>\n",
"21291.0\n",
"</td>\n",
"<td>\n",
"21220.5\n",
"</td>\n",
"<td>\n",
"21272.5\n",
"</td>\n",
"<td>\n",
"2342.05\n",
"</td>\n",
"</tr>\n",
"</tbody>\n",
"</table>\n",
"</div>"
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"## 【5. 検証結果】"
],
"metadata": {
"id": "skZRK8B9hgIo"
}
},
{
"cell_type": "code",
"source": [
"print(f'[約定履歴CSV]: {FILE_PATH}')\n",
"print(f'[レコード数 ]: {csv_records:,} rows')\n",
"print(f'[CSVサイズ ]: {csv_size:,} MB')\n",
"\n",
"result['速度比'] = round(result['pandas'] / result['polars'], 2)\n",
"display(result)\n",
"print('※処理時間はミリ秒')"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 279
},
"id": "nsnj7UktYTaR",
"outputId": "1517417b-d3e5-4b13-df5e-803a5035078e"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"[約定履歴CSV]: ./trades_BTCUSDT_2023-01-18.csv\n",
"[レコード数 ]: 1,386,138 rows\n",
"[CSVサイズ ]: 169.668 MB\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" pandas polars 速度比\n",
"検証 \n",
"CSV読み込み 1997.358 917.156 2.18\n",
"約定履歴->OHLCV 1S 115.377 44.667 2.58\n",
"約定履歴->OHLC 売買別出来高 443.944 87.974 5.05\n",
"OHLCV 1S->1H 13.292 2.619 5.08"
],
"text/html": [
"\n",
" <div id=\"df-4a431dfe-5e7a-4e34-bcd9-3ca339662594\">\n",
" <div class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>pandas</th>\n",
" <th>polars</th>\n",
" <th>速度比</th>\n",
" </tr>\n",
" <tr>\n",
" <th>検証</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>CSV読み込み</th>\n",
" <td>1997.358</td>\n",
" <td>917.156</td>\n",
" <td>2.18</td>\n",
" </tr>\n",
" <tr>\n",
" <th>約定履歴-&gt;OHLCV 1S</th>\n",
" <td>115.377</td>\n",
" <td>44.667</td>\n",
" <td>2.58</td>\n",
" </tr>\n",
" <tr>\n",
" <th>約定履歴-&gt;OHLC 売買別出来高</th>\n",
" <td>443.944</td>\n",
" <td>87.974</td>\n",
" <td>5.05</td>\n",
" </tr>\n",
" <tr>\n",
" <th>OHLCV 1S-&gt;1H</th>\n",
" <td>13.292</td>\n",
" <td>2.619</td>\n",
" <td>5.08</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4a431dfe-5e7a-4e34-bcd9-3ca339662594')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
" \n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
" <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
" </svg>\n",
" </button>\n",
" \n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" flex-wrap:wrap;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-4a431dfe-5e7a-4e34-bcd9-3ca339662594 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-4a431dfe-5e7a-4e34-bcd9-3ca339662594');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
" </div>\n",
" "
]
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"※処理時間はミリ秒\n"
]
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment