Last active
February 21, 2023 23:06
-
-
Save kevinjnguyen/8e95fbc48950ad8f809e0ceca32c95ff to your computer and use it in GitHub Desktop.
Notebook gist for nbviewer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "id": "81791ea0", | |
| "metadata": {}, | |
| "source": [ | |
| "# Kaskada with CSV\n", | |
| "\n", | |
| "**Goal:** Demonstrate the new feature work for Kaskada to operate on CSV data. This work is scheduled for demo on 2/28/2023." | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "4b7fe26d", | |
| "metadata": {}, | |
| "source": [ | |
| "## Generating CSV Data\n", | |
| "\n", | |
| "The section below generates a basic dataset." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "id": "c8932954", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "import random\n", | |
| "import names\n", | |
| "import pandas\n", | |
| "\n", | |
| "def generate_dataset(num_members, num_records_per_member):\n", | |
| " member_ids = list(range(0, num_members))\n", | |
| " column_1_name = 'amount'\n", | |
| " column_2_name = 'random_col'\n", | |
| " records = []\n", | |
| " for member_id in member_ids:\n", | |
| " for i in range(0, num_records_per_member):\n", | |
| " records.append({\n", | |
| " 'id': member_id,\n", | |
| " 'time': random.randint(1000000000000000, 9000000000000000), # number of seconds from epoch\n", | |
| " 'name': f\"my-cool-name-{random.randint(-100, 100)}\",\n", | |
| " column_1_name : random.randint(-100, 100),\n", | |
| " column_2_name : f\"some-value-{random.randint(0, 100)}\"\n", | |
| " })\n", | |
| "\n", | |
| " df = pandas.DataFrame(records)\n", | |
| " df['time']= pandas.to_datetime(df['time'])\n", | |
| " return df" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 2, | |
| "id": "31de4153", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div>\n", | |
| "<style scoped>\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style>\n", | |
| "<table border=\"1\" class=\"dataframe\">\n", | |
| " <thead>\n", | |
| " <tr style=\"text-align: right;\">\n", | |
| " <th></th>\n", | |
| " <th>id</th>\n", | |
| " <th>time</th>\n", | |
| " <th>name</th>\n", | |
| " <th>amount</th>\n", | |
| " <th>random_col</th>\n", | |
| " </tr>\n", | |
| " </thead>\n", | |
| " <tbody>\n", | |
| " <tr>\n", | |
| " <th>0</th>\n", | |
| " <td>0</td>\n", | |
| " <td>1970-03-17 08:05:44.054039402</td>\n", | |
| " <td>my-cool-name-98</td>\n", | |
| " <td>-11</td>\n", | |
| " <td>some-value-91</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>1</th>\n", | |
| " <td>0</td>\n", | |
| " <td>1970-02-03 11:37:23.437344037</td>\n", | |
| " <td>my-cool-name--36</td>\n", | |
| " <td>17</td>\n", | |
| " <td>some-value-21</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>2</th>\n", | |
| " <td>0</td>\n", | |
| " <td>1970-02-08 18:23:47.317902614</td>\n", | |
| " <td>my-cool-name-72</td>\n", | |
| " <td>-37</td>\n", | |
| " <td>some-value-91</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>3</th>\n", | |
| " <td>0</td>\n", | |
| " <td>1970-03-26 20:19:55.763123946</td>\n", | |
| " <td>my-cool-name-80</td>\n", | |
| " <td>-56</td>\n", | |
| " <td>some-value-64</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>4</th>\n", | |
| " <td>0</td>\n", | |
| " <td>1970-01-14 15:05:01.532420077</td>\n", | |
| " <td>my-cool-name--76</td>\n", | |
| " <td>-46</td>\n", | |
| " <td>some-value-37</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>...</th>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " <td>...</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>495</th>\n", | |
| " <td>99</td>\n", | |
| " <td>1970-01-28 18:05:50.734926519</td>\n", | |
| " <td>my-cool-name--74</td>\n", | |
| " <td>-94</td>\n", | |
| " <td>some-value-50</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>496</th>\n", | |
| " <td>99</td>\n", | |
| " <td>1970-04-01 19:21:15.499511455</td>\n", | |
| " <td>my-cool-name--56</td>\n", | |
| " <td>20</td>\n", | |
| " <td>some-value-11</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>497</th>\n", | |
| " <td>99</td>\n", | |
| " <td>1970-01-23 22:59:31.311351492</td>\n", | |
| " <td>my-cool-name-39</td>\n", | |
| " <td>51</td>\n", | |
| " <td>some-value-62</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>498</th>\n", | |
| " <td>99</td>\n", | |
| " <td>1970-02-15 05:19:26.916549373</td>\n", | |
| " <td>my-cool-name--71</td>\n", | |
| " <td>78</td>\n", | |
| " <td>some-value-30</td>\n", | |
| " </tr>\n", | |
| " <tr>\n", | |
| " <th>499</th>\n", | |
| " <td>99</td>\n", | |
| " <td>1970-01-17 15:34:19.913878785</td>\n", | |
| " <td>my-cool-name-88</td>\n", | |
| " <td>28</td>\n", | |
| " <td>some-value-28</td>\n", | |
| " </tr>\n", | |
| " </tbody>\n", | |
| "</table>\n", | |
| "<p>500 rows × 5 columns</p>\n", | |
| "</div>" | |
| ], | |
| "text/plain": [ | |
| " id time name amount random_col\n", | |
| "0 0 1970-03-17 08:05:44.054039402 my-cool-name-98 -11 some-value-91\n", | |
| "1 0 1970-02-03 11:37:23.437344037 my-cool-name--36 17 some-value-21\n", | |
| "2 0 1970-02-08 18:23:47.317902614 my-cool-name-72 -37 some-value-91\n", | |
| "3 0 1970-03-26 20:19:55.763123946 my-cool-name-80 -56 some-value-64\n", | |
| "4 0 1970-01-14 15:05:01.532420077 my-cool-name--76 -46 some-value-37\n", | |
| ".. .. ... ... ... ...\n", | |
| "495 99 1970-01-28 18:05:50.734926519 my-cool-name--74 -94 some-value-50\n", | |
| "496 99 1970-04-01 19:21:15.499511455 my-cool-name--56 20 some-value-11\n", | |
| "497 99 1970-01-23 22:59:31.311351492 my-cool-name-39 51 some-value-62\n", | |
| "498 99 1970-02-15 05:19:26.916549373 my-cool-name--71 78 some-value-30\n", | |
| "499 99 1970-01-17 15:34:19.913878785 my-cool-name-88 28 some-value-28\n", | |
| "\n", | |
| "[500 rows x 5 columns]" | |
| ] | |
| }, | |
| "execution_count": 2, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "dataset1 = generate_dataset(100, 5)\n", | |
| "dataset1.to_csv('dataset1.csv')\n", | |
| "dataset1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "8f66bbfa", | |
| "metadata": {}, | |
| "source": [ | |
| "## Launch Kaskada\n", | |
| "\n", | |
| "There is no additional configuration Kaskada needs to utilize CSV. Simply create a table or use an existing table, and load the CSV data to the table. Previously, the python client library used PyArrow to convert CSV to Parquet prior to ingestion but now this constraint is removed." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 3, | |
| "id": "f121f541", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "INFO:kaskada.api.session:Initializing manager process\n", | |
| "INFO:kaskada.api.session:Initializing compute process\n", | |
| "INFO:kaskada.api.session:Successfully connected to session.\n" | |
| ] | |
| }, | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<style>table.kda_table {\n", | |
| " border: 1px solid grey !important;\n", | |
| " border-collapse: collapse !important;\n", | |
| " border-spacing: 0 !important;\n", | |
| " font-size: 14px !important;\n", | |
| " margin-bottom: 6px !important;\n", | |
| "}\n", | |
| "\n", | |
| "table.kda_table table {\n", | |
| " margin-bottom: 0 !important;\n", | |
| "}\n", | |
| "\n", | |
| ".kda_table td, .kda_table th {\n", | |
| " border: 1px solid grey !important;\n", | |
| " border-collapse: collapse !important;\n", | |
| " padding: 6px !important;\n", | |
| " text-align: left !important;\n", | |
| " vertical-align: top !important;\n", | |
| "}\n", | |
| "\n", | |
| ".kda_table tr {\n", | |
| " background-color: inherit !important;\n", | |
| "}\n", | |
| "\n", | |
| "/* Style the tab */\n", | |
| ".kda_tab {\n", | |
| " overflow: hidden;\n", | |
| " border-bottom: 1px solid grey;\n", | |
| " background-color: inherit;\n", | |
| "}\n", | |
| "\n", | |
| "/* Style the buttons that are used to open the tab content */\n", | |
| ".kda_tab button {\n", | |
| " background-color: inherit;\n", | |
| " border-top: 1px solid grey;\n", | |
| " border-right: 1px solid grey;\n", | |
| " border-left: 1px solid grey;\n", | |
| " border-bottom: none;\n", | |
| " border-radius: 5px 5px 0px 0px;\n", | |
| " color: inherit;\n", | |
| " cursor: pointer;\n", | |
| " float: left;\n", | |
| " margin: 2px 8px 0px 2px;\n", | |
| " outline: none;\n", | |
| " padding: 5px 16px;\n", | |
| " transition: 0.3s;\n", | |
| "}\n", | |
| "\n", | |
| "/* Change background color of buttons on hover */\n", | |
| ".kda_tab button:hover {\n", | |
| " background-color: grey;\n", | |
| "}\n", | |
| "\n", | |
| "/* Create an active/current tablink class */\n", | |
| ".kda_tab button.kda_active {\n", | |
| " outline: 2px solid grey;\n", | |
| "}\n", | |
| "\n", | |
| "/* Style the tab content */\n", | |
| ".kda_tabcontent {\n", | |
| " display: none;\n", | |
| " padding: 6px;\n", | |
| " border: 1px solid grey;\n", | |
| " border-top: none;\n", | |
| "}\n", | |
| "\n", | |
| ".kda_scroll_container {\n", | |
| " max-height: 400px;\n", | |
| " overflow: auto;\n", | |
| "}\n", | |
| "</style>" | |
| ], | |
| "text/plain": [ | |
| "<IPython.core.display.HTML object>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "output_type": "display_data" | |
| }, | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<script type=\"text/javascript\">function openTab(evt, tabName) {\n", | |
| " // Declare all variables\n", | |
| " var i, rootElement, tabcontent, tablinks, tabshow;\n", | |
| "\n", | |
| " // Get all elements with class=\"tabcontent\" and hide them\n", | |
| " rootElement = evt.currentTarget.parentElement.parentElement;\n", | |
| " tabcontent = rootElement.getElementsByClassName(\"kda_tabcontent\");\n", | |
| " for (i = 0; i < tabcontent.length; i++) {\n", | |
| " tabcontent[i].style.display = \"none\";\n", | |
| " }\n", | |
| "\n", | |
| " // Get all elements with class=\"tablinks\" and remove the class \"active\"\n", | |
| " tablinks = rootElement.getElementsByClassName(\"kda_tablinks\");\n", | |
| " for (i = 0; i < tablinks.length; i++) {\n", | |
| " tablinks[i].className = tablinks[i].className.replace(\" kda_active\", \"\");\n", | |
| " }\n", | |
| "\n", | |
| " // Show the current tab, and add an \"active\" class to the button that opened the tab\n", | |
| " tabshow = rootElement.getElementsByClassName(tabName);\n", | |
| " for (i = 0; i < tabshow.length; i++) {\n", | |
| " tabshow[i].style.display = \"block\";\n", | |
| " }\n", | |
| " evt.currentTarget.className += \" kda_active\";\n", | |
| "}\n", | |
| "</script>" | |
| ], | |
| "text/plain": [ | |
| "<IPython.core.display.HTML object>" | |
| ] | |
| }, | |
| "metadata": {}, | |
| "output_type": "display_data" | |
| } | |
| ], | |
| "source": [ | |
| "from kaskada.api.session import LocalBuilder\n", | |
| "import kaskada.table\n", | |
| "session = LocalBuilder().download(False).build()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 4, | |
| "id": "dace5ed5", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"details kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>table</b></td><td><table class=\"kda_table\"><tr><td><b>table_name</b></td><td><pre>transactions</pre></td></tr><tr><td><b>entity_key_column_name</b></td><td><pre>name</pre></td></tr><tr><td><b>time_column_name</b></td><td><pre>time</pre></td></tr><tr><td><b>version</b></td><td><pre>0</pre></td></tr><tr><td><b>create_time</b></td><td><pre>2023-02-21T14:30:18.601672</pre></td></tr><tr><td><b>update_time</b></td><td><pre>2023-02-21T14:30:18.601673</pre></td></tr></table></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>dc6b866f460e95cda72ade26faeb8e21</pre></td></tr></table></td></tr></table></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>table {\n", | |
| " table_id: \"ad3c912b-aa97-4c99-b5d3-678efe30aab6\"\n", | |
| " table_name: \"transactions\"\n", | |
| " time_column_name: \"time\"\n", | |
| " entity_key_column_name: \"name\"\n", | |
| " create_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 601672000\n", | |
| " }\n", | |
| " update_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 601673000\n", | |
| " }\n", | |
| " table_source {\n", | |
| " kaskada {\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"dc6b866f460e95cda72ade26faeb8e21\"\n", | |
| "}\n", | |
| "</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "table {\n", | |
| " table_id: \"ad3c912b-aa97-4c99-b5d3-678efe30aab6\"\n", | |
| " table_name: \"transactions\"\n", | |
| " time_column_name: \"time\"\n", | |
| " entity_key_column_name: \"name\"\n", | |
| " create_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 601672000\n", | |
| " }\n", | |
| " update_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 601673000\n", | |
| " }\n", | |
| " table_source {\n", | |
| " kaskada {\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"dc6b866f460e95cda72ade26faeb8e21\"\n", | |
| "}" | |
| ] | |
| }, | |
| "execution_count": 4, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "# Create the table named transactions with the time and name column\n", | |
| "kaskada.table.create_table('transactions', 'time', 'name')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 5, | |
| "id": "40008105", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"details kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>data_token_id</b></td><td><pre>3ec1e730-e169-4a4b-ac86-716e57fa6b9f</pre></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>5574a95ec517ce47725a88fc4fc7149a</pre></td></tr></table></td></tr></table></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>data_token_id: \"3ec1e730-e169-4a4b-ac86-716e57fa6b9f\"\n", | |
| "request_details {\n", | |
| " request_id: \"5574a95ec517ce47725a88fc4fc7149a\"\n", | |
| "}\n", | |
| "</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "data_token_id: \"3ec1e730-e169-4a4b-ac86-716e57fa6b9f\"\n", | |
| "request_details {\n", | |
| " request_id: \"5574a95ec517ce47725a88fc4fc7149a\"\n", | |
| "}" | |
| ] | |
| }, | |
| "execution_count": 5, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "# Load the data to the table\n", | |
| "kaskada.table.load('transactions', 'dataset1.csv')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 6, | |
| "id": "c0752147", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'schema')\">Schema</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"details kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>table</b></td><td><table class=\"kda_table\"><tr><td><b>table_name</b></td><td><pre>transactions</pre></td></tr><tr><td><b>entity_key_column_name</b></td><td><pre>name</pre></td></tr><tr><td><b>time_column_name</b></td><td><pre>time</pre></td></tr><tr><td><b>version</b></td><td><pre>1</pre></td></tr><tr><td><b>schema</b></td><td><pre>(see Schema tab)</pre></td></tr><tr><td><b>create_time</b></td><td><pre>2023-02-21T14:30:18.601672</pre></td></tr><tr><td><b>update_time</b></td><td><pre>2023-02-21T14:30:18.646153</pre></td></tr></table></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>c99e070c60af2b12dc3ff22da9d18f18</pre></td></tr></table></td></tr></table></div></div><div class=\"schema kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>column_name</th><th>column_type</th></tr></thead><tbody><tr><th>1</th><td>id</td><td>i64</td></tr><tr><th>2</th><td>time</td><td>timestamp_ns</td></tr><tr><th>3</th><td>name</td><td>string</td></tr><tr><th>4</th><td>amount</td><td>i64</td></tr><tr><th>5</th><td>random_col</td><td>string</td></tr></tbody></table></div></body></html></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>table {\n", | |
| " table_id: \"ad3c912b-aa97-4c99-b5d3-678efe30aab6\"\n", | |
| " table_name: \"transactions\"\n", | |
| " time_column_name: \"time\"\n", | |
| " entity_key_column_name: \"name\"\n", | |
| " create_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 601672000\n", | |
| " }\n", | |
| " update_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 646153000\n", | |
| " }\n", | |
| " version: 1\n", | |
| " schema {\n", | |
| " fields {\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"id\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"time\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_TIMESTAMP_NANOSECOND\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"name\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"amount\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"random_col\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| " table_source {\n", | |
| " kaskada {\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"c99e070c60af2b12dc3ff22da9d18f18\"\n", | |
| "}\n", | |
| "</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "table {\n", | |
| " table_id: \"ad3c912b-aa97-4c99-b5d3-678efe30aab6\"\n", | |
| " table_name: \"transactions\"\n", | |
| " time_column_name: \"time\"\n", | |
| " entity_key_column_name: \"name\"\n", | |
| " create_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 601672000\n", | |
| " }\n", | |
| " update_time {\n", | |
| " seconds: 1677011418\n", | |
| " nanos: 646153000\n", | |
| " }\n", | |
| " version: 1\n", | |
| " schema {\n", | |
| " fields {\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"id\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"time\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_TIMESTAMP_NANOSECOND\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"name\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"amount\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"random_col\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| " table_source {\n", | |
| " kaskada {\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"c99e070c60af2b12dc3ff22da9d18f18\"\n", | |
| "}" | |
| ] | |
| }, | |
| "execution_count": 6, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "# Get the table and see the version is incremented and the schema is available.\n", | |
| "kaskada.table.get_table('transactions')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "852448de", | |
| "metadata": {}, | |
| "source": [ | |
| "## Run a query with Fenlmagic" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 7, | |
| "id": "0a6c2f4c", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "%load_ext fenlmagic" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 8, | |
| "id": "0b8ae92f", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'dataframe')\">Dataframe</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'schema')\">Schema</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"dataframe kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>_time</th><th>_subsort</th><th>_key_hash</th><th>_key</th><th></th><th>id</th><th>time</th><th>name</th><th>amount</th><th>random_col</th></tr></thead><tbody><tr><th>0</th><td>1970-01-13 03:28:13.728971017</td><td>2015731755919230445</td><td>17784318451884400532</td><td>my-cool-name--60</td><td>131</td><td>26</td><td>1970-01-13 03:28:13.728971017</td><td>my-cool-name--60</td><td>-6</td><td>some-value-12</td></tr><tr><th>1</th><td>1970-01-13 05:34:43.465807800</td><td>2015731755919230794</td><td>13421628122206835095</td><td>my-cool-name-61</td><td>480</td><td>96</td><td>1970-01-13 05:34:43.465807800</td><td>my-cool-name-61</td><td>29</td><td>some-value-1</td></tr><tr><th>2</th><td>1970-01-13 13:33:11.182235465</td><td>2015731755919230802</td><td>13929080721858716940</td><td>my-cool-name-92</td><td>488</td><td>97</td><td>1970-01-13 13:33:11.182235465</td><td>my-cool-name-92</td><td>65</td><td>some-value-48</td></tr><tr><th>3</th><td>1970-01-13 14:50:14.998700322</td><td>2015731755919230412</td><td>16536877432099839762</td><td>my-cool-name-10</td><td>98</td><td>19</td><td>1970-01-13 14:50:14.998700322</td><td>my-cool-name-10</td><td>-92</td><td>some-value-48</td></tr><tr><th>4</th><td>1970-01-13 22:42:18.776714240</td><td>2015731755919230799</td><td>9592677144762243636</td><td>my-cool-name-84</td><td>485</td><td>97</td><td>1970-01-13 22:42:18.776714240</td><td>my-cool-name-84</td><td>-91</td><td>some-value-27</td></tr><tr><th>...</th><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr><tr><th>495</th><td>1970-04-13 11:01:43.518846396</td><td>2015731755919230601</td><td>17107532125463420378</td><td>my-cool-name--22</td><td>287</td><td>57</td><td>1970-04-13 11:01:43.518846396</td><td>my-cool-name--22</td><td>22</td><td>some-value-19</td></tr><tr><th>496</th><td>1970-04-14 10:58:09.370409983</td><td>2015731755919230595</td><td>12451476979163753387</td><td>my-cool-name-89</td><td>281</td><td>56</td><td>1970-04-14 10:58:09.370409983</td><td>my-cool-name-89</td><td>-34</td><td>some-value-91</td></tr><tr><th>497</th><td>1970-04-14 14:40:24.437678893</td><td>2015731755919230569</td><td>10760766651922208188</td><td>my-cool-name--69</td><td>255</td><td>51</td><td>1970-04-14 14:40:24.437678893</td><td>my-cool-name--69</td><td>32</td><td>some-value-91</td></tr><tr><th>498</th><td>1970-04-14 19:16:28.600806925</td><td>2015731755919230621</td><td>17639341337839854446</td><td>my-cool-name-33</td><td>307</td><td>61</td><td>1970-04-14 19:16:28.600806925</td><td>my-cool-name-33</td><td>-98</td><td>some-value-53</td></tr><tr><th>499</th><td>1970-04-14 21:54:50.526956982</td><td>2015731755919230365</td><td>4486326919160747726</td><td>my-cool-name--74</td><td>51</td><td>10</td><td>1970-04-14 21:54:50.526956982</td><td>my-cool-name--74</td><td>38</td><td>some-value-60</td></tr></tbody></table><p>500 rows × 10 columns</p></div></body></html></div></div><div class=\"details kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>state</b></td><td>SUCCESS</td></tr><tr><td><b>query_id</b></td><td>30e93c69-a6c6-4b74-92f7-305fddfaea38</td></tr><tr><td><b>metrics</b></td><td><table class=\"kda_table\"><tr><td><b>time_preparing</b></td><td>0.032s</td></tr><tr><td><b>time_computing</b></td><td>0.005s</td></tr><tr><td><b>output_files</b></td><td>1</td></tr></table></td></tr><tr><td><b>file_results</b></td><td><table class=\"kda_table\"><tr><th>index</th><th>path</th></tr><tr><td><pre>0</pre></td><td><a href=\"/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/41fd75ce-0864-4c62-aa7a-7022836ce8c7.parquet\">/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/41fd75ce-0864-4c62-aa7a-7022836ce8c7.parquet</a></td></tr></table></td></tr><tr><td><b>analysis</b></td><td><table class=\"kda_table\"><tr><td><b>can_execute</b></td><td><pre>True</pre></td></tr></table></td></tr><tr><td><b>schema</b></td><td><pre>(see Schema tab)</pre></td></tr><tr><td><b>data_token_id</b></td><td><pre>3ec1e730-e169-4a4b-ac86-716e57fa6b9f</pre></td></tr><tr><td><b>slice_request</b></td><td><table class=\"kda_table\"><tr><td><b>None</b></td><td><pre>(full dataset used for query)</pre></td></tr></table></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>9356ee432711847485c67b9e8c3f9eff</pre></td></tr></table></td></tr><tr><td><b>query</b></td><td><pre>transactions</pre></td></tr></table></div></div><div class=\"schema kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>column_name</th><th>column_type</th></tr></thead><tbody><tr><th>1</th><td>id</td><td>i64</td></tr><tr><th>2</th><td>time</td><td>timestamp_ns</td></tr><tr><th>3</th><td>name</td><td>string</td></tr><tr><th>4</th><td>amount</td><td>i64</td></tr><tr><th>5</th><td>random_col</td><td>string</td></tr></tbody></table></div></body></html></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>{'dataframe': 'RangeIndex: 500 entries, 0 to 499'\n", | |
| " 'Columns: 10 entries, _time to random_col'\n", | |
| " 'dtypes: datetime64[ns](2), int64(3), object(3), uint64(2)'\n", | |
| " 'memory usage: 39.2+ KB',\n", | |
| " 'query': 'transactions',\n", | |
| " 'query_response': state: STATE_SUCCESS\n", | |
| "config {\n", | |
| " data_token_id: \"3ec1e730-e169-4a4b-ac86-716e57fa6b9f\"\n", | |
| "}\n", | |
| "analysis {\n", | |
| " can_execute: true\n", | |
| " schema {\n", | |
| " fields {\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"id\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"time\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_TIMESTAMP_NANOSECOND\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"name\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"amount\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"random_col\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "fenl_diagnostics {\n", | |
| "}\n", | |
| "metrics {\n", | |
| " time_preparing {\n", | |
| " nanos: 32682666\n", | |
| " }\n", | |
| " time_computing {\n", | |
| " nanos: 5978750\n", | |
| " }\n", | |
| " output_files: 1\n", | |
| " total_input_rows: 500\n", | |
| " processed_input_rows: 500\n", | |
| " produced_output_rows: 500\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"9356ee432711847485c67b9e8c3f9eff\"\n", | |
| "}\n", | |
| "query_id: \"30e93c69-a6c6-4b74-92f7-305fddfaea38\"\n", | |
| "file_results {\n", | |
| " file_type: FILE_TYPE_PARQUET\n", | |
| " paths: \"/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/41fd75ce-0864-4c62-aa7a-7022836ce8c7.parquet\"\n", | |
| "}\n", | |
| "}</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "<fenlmagic.QueryResult at 0x127fc4460>" | |
| ] | |
| }, | |
| "execution_count": 8, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "%%fenl\n", | |
| "transactions" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "id": "e4c95917", | |
| "metadata": {}, | |
| "source": [ | |
| "## More Data\n", | |
| "\n", | |
| "Add more data with additional CSV files." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 9, | |
| "id": "894d2d61", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "dataset2 = generate_dataset(100, 50) #5000\n", | |
| "dataset2.to_csv('dataset2.csv')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 10, | |
| "id": "a22a72a9", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"details kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>data_token_id</b></td><td><pre>5679dc97-edfe-4f46-82a9-154579ae2bf8</pre></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>e4f3ab89dd6cd3bfd7dfb94134e50032</pre></td></tr></table></td></tr></table></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>data_token_id: \"5679dc97-edfe-4f46-82a9-154579ae2bf8\"\n", | |
| "request_details {\n", | |
| " request_id: \"e4f3ab89dd6cd3bfd7dfb94134e50032\"\n", | |
| "}\n", | |
| "</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "data_token_id: \"5679dc97-edfe-4f46-82a9-154579ae2bf8\"\n", | |
| "request_details {\n", | |
| " request_id: \"e4f3ab89dd6cd3bfd7dfb94134e50032\"\n", | |
| "}" | |
| ] | |
| }, | |
| "execution_count": 10, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "kaskada.table.load('transactions', 'dataset2.csv')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 11, | |
| "id": "1ba89181", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'dataframe')\">Dataframe</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'schema')\">Schema</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"dataframe kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>_time</th><th>_subsort</th><th>_key_hash</th><th>_key</th><th></th><th>id</th><th>time</th><th>name</th><th>amount</th><th>random_col</th></tr></thead><tbody><tr><th>0</th><td>1970-01-12 14:50:24.465558510</td><td>8929113797628956394</td><td>16625143277736426487</td><td>my-cool-name-70</td><td>4224</td><td>84</td><td>1970-01-12 14:50:24.465558510</td><td>my-cool-name-70</td><td>-78</td><td>some-value-19</td></tr><tr><th>1</th><td>1970-01-12 15:08:17.339553030</td><td>8929113797628955225</td><td>12415044456207887489</td><td>my-cool-name--8</td><td>3055</td><td>61</td><td>1970-01-12 15:08:17.339553030</td><td>my-cool-name--8</td><td>60</td><td>some-value-26</td></tr><tr><th>2</th><td>1970-01-12 15:25:19.102596796</td><td>8929113797628957064</td><td>13661589225801312423</td><td>my-cool-name--35</td><td>4894</td><td>97</td><td>1970-01-12 15:25:19.102596796</td><td>my-cool-name--35</td><td>17</td><td>some-value-47</td></tr><tr><th>3</th><td>1970-01-12 15:54:34.406636374</td><td>8929113797628956666</td><td>9208299946578562835</td><td>my-cool-name--9</td><td>4496</td><td>89</td><td>1970-01-12 15:54:34.406636374</td><td>my-cool-name--9</td><td>9</td><td>some-value-58</td></tr><tr><th>4</th><td>1970-01-12 16:16:14.728824083</td><td>8929113797628955356</td><td>9208299946578562835</td><td>my-cool-name--9</td><td>3186</td><td>63</td><td>1970-01-12 16:16:14.728824083</td><td>my-cool-name--9</td><td>-89</td><td>some-value-31</td></tr><tr><th>...</th><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr><tr><th>5495</th><td>1970-04-15 01:47:08.914068320</td><td>8929113797628956338</td><td>7307995790805233380</td><td>my-cool-name-67</td><td>4168</td><td>83</td><td>1970-04-15 01:47:08.914068320</td><td>my-cool-name-67</td><td>-90</td><td>some-value-38</td></tr><tr><th>5496</th><td>1970-04-15 02:48:47.014722253</td><td>8929113797628955535</td><td>15888279268039338099</td><td>my-cool-name-38</td><td>3365</td><td>67</td><td>1970-04-15 02:48:47.014722253</td><td>my-cool-name-38</td><td>-85</td><td>some-value-33</td></tr><tr><th>5497</th><td>1970-04-15 02:54:14.223501314</td><td>8929113797628955969</td><td>18332192212608896962</td><td>my-cool-name-47</td><td>3799</td><td>75</td><td>1970-04-15 02:54:14.223501314</td><td>my-cool-name-47</td><td>-35</td><td>some-value-39</td></tr><tr><th>5498</th><td>1970-04-15 03:39:31.213555781</td><td>8929113797628953908</td><td>51216841596388751</td><td>my-cool-name-25</td><td>1738</td><td>34</td><td>1970-04-15 03:39:31.213555781</td><td>my-cool-name-25</td><td>-64</td><td>some-value-16</td></tr><tr><th>5499</th><td>1970-04-15 03:50:54.825574575</td><td>8929113797628955592</td><td>10607954918054055479</td><td>my-cool-name--76</td><td>3422</td><td>68</td><td>1970-04-15 03:50:54.825574575</td><td>my-cool-name--76</td><td>37</td><td>some-value-76</td></tr></tbody></table><p>5500 rows × 10 columns</p></div></body></html></div></div><div class=\"details kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>state</b></td><td>SUCCESS</td></tr><tr><td><b>query_id</b></td><td>70c1efe8-9e92-4026-8a7c-586fcefd4144</td></tr><tr><td><b>metrics</b></td><td><table class=\"kda_table\"><tr><td><b>time_preparing</b></td><td>0.245s</td></tr><tr><td><b>time_computing</b></td><td>0.007s</td></tr><tr><td><b>output_files</b></td><td>1</td></tr></table></td></tr><tr><td><b>file_results</b></td><td><table class=\"kda_table\"><tr><th>index</th><th>path</th></tr><tr><td><pre>0</pre></td><td><a href=\"/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/28e07007-1fc5-4f80-b390-8f6917c5824a.parquet\">/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/28e07007-1fc5-4f80-b390-8f6917c5824a.parquet</a></td></tr></table></td></tr><tr><td><b>analysis</b></td><td><table class=\"kda_table\"><tr><td><b>can_execute</b></td><td><pre>True</pre></td></tr></table></td></tr><tr><td><b>schema</b></td><td><pre>(see Schema tab)</pre></td></tr><tr><td><b>data_token_id</b></td><td><pre>5679dc97-edfe-4f46-82a9-154579ae2bf8</pre></td></tr><tr><td><b>slice_request</b></td><td><table class=\"kda_table\"><tr><td><b>None</b></td><td><pre>(full dataset used for query)</pre></td></tr></table></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>e58f9b12864ee421e96a219d9b3488f7</pre></td></tr></table></td></tr><tr><td><b>query</b></td><td><pre>transactions</pre></td></tr></table></div></div><div class=\"schema kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>column_name</th><th>column_type</th></tr></thead><tbody><tr><th>1</th><td>id</td><td>i64</td></tr><tr><th>2</th><td>time</td><td>timestamp_ns</td></tr><tr><th>3</th><td>name</td><td>string</td></tr><tr><th>4</th><td>amount</td><td>i64</td></tr><tr><th>5</th><td>random_col</td><td>string</td></tr></tbody></table></div></body></html></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>{'dataframe': 'RangeIndex: 5500 entries, 0 to 5499'\n", | |
| " 'Columns: 10 entries, _time to random_col'\n", | |
| " 'dtypes: datetime64[ns](2), int64(3), object(3), uint64(2)'\n", | |
| " 'memory usage: 429.8+ KB',\n", | |
| " 'query': 'transactions',\n", | |
| " 'query_response': state: STATE_SUCCESS\n", | |
| "config {\n", | |
| " data_token_id: \"5679dc97-edfe-4f46-82a9-154579ae2bf8\"\n", | |
| "}\n", | |
| "analysis {\n", | |
| " can_execute: true\n", | |
| " schema {\n", | |
| " fields {\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"id\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"time\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_TIMESTAMP_NANOSECOND\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"name\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"amount\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"random_col\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "fenl_diagnostics {\n", | |
| "}\n", | |
| "metrics {\n", | |
| " time_preparing {\n", | |
| " nanos: 245954292\n", | |
| " }\n", | |
| " time_computing {\n", | |
| " nanos: 7721542\n", | |
| " }\n", | |
| " output_files: 1\n", | |
| " total_input_rows: 5500\n", | |
| " processed_input_rows: 5500\n", | |
| " produced_output_rows: 5500\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"e58f9b12864ee421e96a219d9b3488f7\"\n", | |
| "}\n", | |
| "query_id: \"70c1efe8-9e92-4026-8a7c-586fcefd4144\"\n", | |
| "file_results {\n", | |
| " file_type: FILE_TYPE_PARQUET\n", | |
| " paths: \"/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/28e07007-1fc5-4f80-b390-8f6917c5824a.parquet\"\n", | |
| "}\n", | |
| "}</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "<fenlmagic.QueryResult at 0x127f419d0>" | |
| ] | |
| }, | |
| "execution_count": 11, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "%%fenl\n", | |
| "transactions" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": 12, | |
| "id": "da4c3955", | |
| "metadata": {}, | |
| "outputs": [ | |
| { | |
| "data": { | |
| "text/html": [ | |
| "<div><div class=\"kda_tab\"><button class=\"kda_tablinks kda_active\" onclick=\"openTab(event, 'dataframe')\">Dataframe</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'details')\">Details</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'schema')\">Schema</button><button class=\"kda_tablinks\" onclick=\"openTab(event, 'raw')\">Raw</button></div><div class=\"dataframe kda_tabcontent\" style=\"display:block;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>_time</th><th>_subsort</th><th>_key_hash</th><th>_key</th><th>Unnamed: 4</th><th>id</th><th>time</th><th>name</th><th>amount</th><th>random_col</th></tr></thead><tbody><tr><th>0</th><td>1970-01-12T14:50:24.465558510</td><td>8929113797628956394</td><td>16625143277736426487</td><td>my-cool-name-70</td><td>4224</td><td>84</td><td>1970-01-12T14:50:24.465558510</td><td>my-cool-name-70</td><td>-78</td><td>some-value-19</td></tr><tr><th>1</th><td>1970-01-12T15:08:17.339553030</td><td>8929113797628955225</td><td>12415044456207887489</td><td>my-cool-name--8</td><td>3055</td><td>61</td><td>1970-01-12T15:08:17.339553030</td><td>my-cool-name--8</td><td>60</td><td>some-value-26</td></tr><tr><th>2</th><td>1970-01-12T15:25:19.102596796</td><td>8929113797628957064</td><td>13661589225801312423</td><td>my-cool-name--35</td><td>4894</td><td>97</td><td>1970-01-12T15:25:19.102596796</td><td>my-cool-name--35</td><td>17</td><td>some-value-47</td></tr><tr><th>3</th><td>1970-01-12T15:54:34.406636374</td><td>8929113797628956666</td><td>9208299946578562835</td><td>my-cool-name--9</td><td>4496</td><td>89</td><td>1970-01-12T15:54:34.406636374</td><td>my-cool-name--9</td><td>9</td><td>some-value-58</td></tr><tr><th>4</th><td>1970-01-12T16:16:14.728824083</td><td>8929113797628955356</td><td>9208299946578562835</td><td>my-cool-name--9</td><td>3186</td><td>63</td><td>1970-01-12T16:16:14.728824083</td><td>my-cool-name--9</td><td>-89</td><td>some-value-31</td></tr><tr><th>...</th><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr><tr><th>5495</th><td>1970-04-15T01:47:08.914068320</td><td>8929113797628956338</td><td>7307995790805233380</td><td>my-cool-name-67</td><td>4168</td><td>83</td><td>1970-04-15T01:47:08.914068320</td><td>my-cool-name-67</td><td>-90</td><td>some-value-38</td></tr><tr><th>5496</th><td>1970-04-15T02:48:47.014722253</td><td>8929113797628955535</td><td>15888279268039338099</td><td>my-cool-name-38</td><td>3365</td><td>67</td><td>1970-04-15T02:48:47.014722253</td><td>my-cool-name-38</td><td>-85</td><td>some-value-33</td></tr><tr><th>5497</th><td>1970-04-15T02:54:14.223501314</td><td>8929113797628955969</td><td>18332192212608896962</td><td>my-cool-name-47</td><td>3799</td><td>75</td><td>1970-04-15T02:54:14.223501314</td><td>my-cool-name-47</td><td>-35</td><td>some-value-39</td></tr><tr><th>5498</th><td>1970-04-15T03:39:31.213555781</td><td>8929113797628953908</td><td>51216841596388751</td><td>my-cool-name-25</td><td>1738</td><td>34</td><td>1970-04-15T03:39:31.213555781</td><td>my-cool-name-25</td><td>-64</td><td>some-value-16</td></tr><tr><th>5499</th><td>1970-04-15T03:50:54.825574575</td><td>8929113797628955592</td><td>10607954918054055479</td><td>my-cool-name--76</td><td>3422</td><td>68</td><td>1970-04-15T03:50:54.825574575</td><td>my-cool-name--76</td><td>37</td><td>some-value-76</td></tr></tbody></table><p>5500 rows × 10 columns</p></div></body></html></div></div><div class=\"details kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><table class=\"kda_table\"><tr><td><b>state</b></td><td>SUCCESS</td></tr><tr><td><b>query_id</b></td><td>77de770e-8905-46ff-ad44-20eb3478cb43</td></tr><tr><td><b>metrics</b></td><td><table class=\"kda_table\"><tr><td><b>time_preparing</b></td><td>0.001s</td></tr><tr><td><b>time_computing</b></td><td>0.014s</td></tr><tr><td><b>output_files</b></td><td>1</td></tr></table></td></tr><tr><td><b>file_results</b></td><td><table class=\"kda_table\"><tr><th>index</th><th>path</th></tr><tr><td><pre>0</pre></td><td><a href=\"/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/13838656-e9aa-4913-95a8-d9dcc6ccfeff.csv\">/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/13838656-e9aa-4913-95a8-d9dcc6ccfeff.csv</a></td></tr></table></td></tr><tr><td><b>analysis</b></td><td><table class=\"kda_table\"><tr><td><b>can_execute</b></td><td><pre>True</pre></td></tr></table></td></tr><tr><td><b>schema</b></td><td><pre>(see Schema tab)</pre></td></tr><tr><td><b>data_token_id</b></td><td><pre>5679dc97-edfe-4f46-82a9-154579ae2bf8</pre></td></tr><tr><td><b>slice_request</b></td><td><table class=\"kda_table\"><tr><td><b>None</b></td><td><pre>(full dataset used for query)</pre></td></tr></table></td></tr><tr><td><b>request_details</b></td><td><table class=\"kda_table\"><tr><td><b>request_id</b></td><td><pre>209e1c8fa4f39a1cc6a38cd63d6b423f</pre></td></tr></table></td></tr><tr><td><b>query</b></td><td><pre>transactions</pre></td></tr></table></div></div><div class=\"schema kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><html><head></head><body><div><style scoped=\"\">\n", | |
| " .dataframe tbody tr th:only-of-type {\n", | |
| " vertical-align: middle;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe tbody tr th {\n", | |
| " vertical-align: top;\n", | |
| " }\n", | |
| "\n", | |
| " .dataframe thead th {\n", | |
| " text-align: right;\n", | |
| " }\n", | |
| "</style><table border=\"1\" class=\"dataframe\"><thead><tr style=\"text-align: right;\"><th></th><th>column_name</th><th>column_type</th></tr></thead><tbody><tr><th>1</th><td>id</td><td>i64</td></tr><tr><th>2</th><td>time</td><td>timestamp_ns</td></tr><tr><th>3</th><td>name</td><td>string</td></tr><tr><th>4</th><td>amount</td><td>i64</td></tr><tr><th>5</th><td>random_col</td><td>string</td></tr></tbody></table></div></body></html></div></div><div class=\"raw kda_tabcontent\" style=\"display:none;\"><div class=\"kda_scroll_container\"><pre>{'dataframe': 'RangeIndex: 5500 entries, 0 to 5499'\n", | |
| " 'Columns: 10 entries, _time to random_col'\n", | |
| " 'dtypes: int64(4), object(5), uint64(1)'\n", | |
| " 'memory usage: 429.8+ KB',\n", | |
| " 'query': 'transactions',\n", | |
| " 'query_response': state: STATE_SUCCESS\n", | |
| "config {\n", | |
| " data_token_id: \"5679dc97-edfe-4f46-82a9-154579ae2bf8\"\n", | |
| "}\n", | |
| "analysis {\n", | |
| " can_execute: true\n", | |
| " schema {\n", | |
| " fields {\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"id\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"time\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_TIMESTAMP_NANOSECOND\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"name\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"amount\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_I64\n", | |
| " }\n", | |
| " }\n", | |
| " fields {\n", | |
| " name: \"random_col\"\n", | |
| " data_type {\n", | |
| " primitive: PRIMITIVE_TYPE_STRING\n", | |
| " }\n", | |
| " }\n", | |
| " }\n", | |
| "}\n", | |
| "fenl_diagnostics {\n", | |
| "}\n", | |
| "metrics {\n", | |
| " time_preparing {\n", | |
| " nanos: 1945167\n", | |
| " }\n", | |
| " time_computing {\n", | |
| " nanos: 14550167\n", | |
| " }\n", | |
| " output_files: 1\n", | |
| " total_input_rows: 5500\n", | |
| " processed_input_rows: 5500\n", | |
| " produced_output_rows: 5500\n", | |
| "}\n", | |
| "request_details {\n", | |
| " request_id: \"209e1c8fa4f39a1cc6a38cd63d6b423f\"\n", | |
| "}\n", | |
| "query_id: \"77de770e-8905-46ff-ad44-20eb3478cb43\"\n", | |
| "file_results {\n", | |
| " file_type: FILE_TYPE_CSV\n", | |
| " paths: \"/Users/kevin.nguyen/Github/kaskada/examples/data/results/2c7b9602-140b-4c1e-8b4c-913260752285/gvWginH1sL6aMQnXpEeGuImTLCw_NxWVb8JJqw/13838656-e9aa-4913-95a8-d9dcc6ccfeff.csv\"\n", | |
| "}\n", | |
| "}</pre></div></div></div>" | |
| ], | |
| "text/plain": [ | |
| "<fenlmagic.QueryResult at 0x107e979d0>" | |
| ] | |
| }, | |
| "execution_count": 12, | |
| "metadata": {}, | |
| "output_type": "execute_result" | |
| } | |
| ], | |
| "source": [ | |
| "%%fenl --output=csv\n", | |
| "transactions" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "id": "eb1caaf9", | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 3 (ipykernel)", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.8.16" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 5 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment