Skip to content

Instantly share code, notes, and snippets.

@Evan-Kim2028

Evan-Kim2028/gist:ff2c752b236ee95d05d56b322ab44f36

Created March 9, 2023 18:00
Show Gist options
  • Save Evan-Kim2028/ff2c752b236ee95d05d56b322ab44f36 to your computer and use it in GitHub Desktop.
Save Evan-Kim2028/ff2c752b236ee95d05d56b322ab44f36 to your computer and use it in GitHub Desktop.
Download ZIP
pagination error with larger query size
Raw
gistfile1.txt
{
"cells": [
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"from subgrounds import Subgrounds\n",
"import polars as pl\n",
"\n",
"from subgrounds.pagination import ShallowStrategy, LegacyStrategy"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# DEFINE TIMESTAMP HERE. Timstamp is used for replication quality assurance purposes.\n",
"timestamp = 1677891498 # current block timestamp is around 1677891498 on March 3rd, 2023 8:06PM\n",
"\n",
"# we set a fixed query size number. The Cow settlements and Uniswap swaps query are multiples larger than this initial query size.\n",
"# query_size = 999 # no pagination error\n",
"query_size = 1500 # pagination error\n",
"\n",
"#Filter size - We filter trades out that are smaller than $1000 USD size\n",
"filter_usd = 1000\n",
"\n",
"token_addr_list = [\n",
" \"0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2\", # weth\n",
" \"0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48\" # usdc\n",
" ]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Messari Univ3 Swaps Schema"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"sg = Subgrounds()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# Load\n",
"cow_sg = sg.load_subgraph('https://api.thegraph.com/subgraphs/name/cowprotocol/cow')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"cow_trades_qp = cow_sg.Query.trades"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"trades_qp = cow_sg.Query.trades(\n",
" orderBy=cow_sg.Query.trades.timestamp,\n",
" orderDirection='desc',\n",
" first=query_size,\n",
" where = {\n",
" 'timestamp_lt': timestamp, \n",
" 'buyAmountUsd_gt': filter_usd, \n",
" 'sellAmountUsd_gt': filter_usd, \n",
" \"sellToken_in\": token_addr_list, \n",
" \"buyToken_in\": token_addr_list\n",
" }\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"trades_df = sg.query_df(trades_qp)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# convert buy and sell amounts to float64\n",
"trades_df['trades_gasPrice'] = trades_df['trades_gasPrice'].astype('float64')\n",
"trades_df['trades_feeAmount'] = trades_df['trades_feeAmount'].astype('float64')\n",
"trades_df['trades_buyAmount'] = trades_df['trades_buyAmount'].astype('float64')\n",
"trades_df['trades_sellAmount'] = trades_df['trades_sellAmount'].astype('float64')"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"# convert trades_df to polars dataframe\n",
"trades_pl = pl.from_pandas(trades_df)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"query returned 1500 rows\n"
]
}
],
"source": [
"print(f'query returned {len(trades_pl)} rows')"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"shape: (2,)\n",
"Series: 'trades_buyToken_id' [str]\n",
"[\n",
"\t\"0xc02aaa39b223...\n",
"\t\"0xa0b86991c621...\n",
"]\n"
]
}
],
"source": [
"# check unique token values\n",
"print(trades_pl['trades_buyToken_id'].unique())"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Cowswap Trades-Settlement Merge"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"# get unique trades_settlement_id list\n",
"trades_settlement_id_list = trades_pl['trades_settlement_id'].unique().to_list()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"# get field path\n",
"settlements_fp = cow_sg.Query.settlements"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# define query path params\n",
"settlements_qp = settlements_fp(\n",
" first=query_size,\n",
" orderBy='firstTradeTimestamp',\n",
" orderDirection='desc',\n",
" where = {'txHash_in': trades_settlement_id_list} \n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"ename": "PaginationError",
"evalue": "Expecting value: line 1 column 1 (char 0)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/requests/models.py:971\u001b[0m, in \u001b[0;36mResponse.json\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 970\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 971\u001b[0m \u001b[39mreturn\u001b[39;00m complexjson\u001b[39m.\u001b[39;49mloads(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtext, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 972\u001b[0m \u001b[39mexcept\u001b[39;00m JSONDecodeError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 973\u001b[0m \u001b[39m# Catch JSON-related errors and raise as requests.JSONDecodeError\u001b[39;00m\n\u001b[1;32m 974\u001b[0m \u001b[39m# This aliases json.JSONDecodeError and simplejson.JSONDecodeError\u001b[39;00m\n",
"File \u001b[0;32m/usr/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[39mif\u001b[39;00m (\u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m parse_float \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_pairs_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[39mreturn\u001b[39;00m _default_decoder\u001b[39m.\u001b[39;49mdecode(s)\n\u001b[1;32m 347\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[0;32m/usr/lib/python3.10/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[39mcontaining a JSON document).\u001b[39;00m\n\u001b[1;32m 335\u001b[0m \n\u001b[1;32m 336\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m obj, end \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mraw_decode(s, idx\u001b[39m=\u001b[39;49m_w(s, \u001b[39m0\u001b[39;49m)\u001b[39m.\u001b[39;49mend())\n\u001b[1;32m 338\u001b[0m end \u001b[39m=\u001b[39m _w(s, end)\u001b[39m.\u001b[39mend()\n",
"File \u001b[0;32m/usr/lib/python3.10/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m \u001b[39mraise\u001b[39;00m JSONDecodeError(\u001b[39m\"\u001b[39m\u001b[39mExpecting value\u001b[39m\u001b[39m\"\u001b[39m, s, err\u001b[39m.\u001b[39mvalue) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39m\n\u001b[1;32m 356\u001b[0m \u001b[39mreturn\u001b[39;00m obj, end\n",
"\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/pagination/pagination.py:79\u001b[0m, in \u001b[0;36mpaginate\u001b[0;34m(schema, doc, pagination_strategy)\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 79\u001b[0m page_data \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49mquery(\n\u001b[1;32m 80\u001b[0m url\u001b[39m=\u001b[39;49mdoc\u001b[39m.\u001b[39;49murl,\n\u001b[1;32m 81\u001b[0m query_str\u001b[39m=\u001b[39;49mdoc\u001b[39m.\u001b[39;49mgraphql,\n\u001b[1;32m 82\u001b[0m variables\u001b[39m=\u001b[39;49mdoc\u001b[39m.\u001b[39;49mvariables \u001b[39m|\u001b[39;49m args\n\u001b[1;32m 83\u001b[0m )\n\u001b[1;32m 84\u001b[0m data \u001b[39m=\u001b[39m merge(data, page_data)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/client.py:162\u001b[0m, in \u001b[0;36mquery\u001b[0;34m(url, query_str, variables)\u001b[0m\n\u001b[1;32m 151\u001b[0m logger\u001b[39m.\u001b[39minfo(\n\u001b[1;32m 152\u001b[0m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mclient.query: url = \u001b[39m\u001b[39m{\u001b[39;00murl\u001b[39m}\u001b[39;00m\u001b[39m, variables = \u001b[39m\u001b[39m{\u001b[39;00mvariables\u001b[39m}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m{\u001b[39;00mquery_str\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\n\u001b[1;32m 153\u001b[0m )\n\u001b[1;32m 154\u001b[0m resp \u001b[39m=\u001b[39m requests\u001b[39m.\u001b[39;49mpost(\n\u001b[1;32m 155\u001b[0m url,\n\u001b[1;32m 156\u001b[0m json\u001b[39m=\u001b[39;49m(\n\u001b[1;32m 157\u001b[0m {\u001b[39m'\u001b[39;49m\u001b[39mquery\u001b[39;49m\u001b[39m'\u001b[39;49m: query_str}\n\u001b[1;32m 158\u001b[0m \u001b[39mif\u001b[39;49;00m variables \u001b[39m==\u001b[39;49m {}\n\u001b[1;32m 159\u001b[0m \u001b[39melse\u001b[39;49;00m {\u001b[39m'\u001b[39;49m\u001b[39mquery\u001b[39;49m\u001b[39m'\u001b[39;49m: query_str, \u001b[39m'\u001b[39;49m\u001b[39mvariables\u001b[39;49m\u001b[39m'\u001b[39;49m: variables}\n\u001b[1;32m 160\u001b[0m ),\n\u001b[1;32m 161\u001b[0m headers\u001b[39m=\u001b[39;49m{\u001b[39m'\u001b[39;49m\u001b[39mContent-Type\u001b[39;49m\u001b[39m'\u001b[39;49m: \u001b[39m'\u001b[39;49m\u001b[39mapplication/json\u001b[39;49m\u001b[39m'\u001b[39;49m}\n\u001b[0;32m--> 162\u001b[0m )\u001b[39m.\u001b[39;49mjson()\n\u001b[1;32m 164\u001b[0m \u001b[39mtry\u001b[39;00m:\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/requests/models.py:975\u001b[0m, in \u001b[0;36mResponse.json\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 972\u001b[0m \u001b[39mexcept\u001b[39;00m JSONDecodeError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 973\u001b[0m \u001b[39m# Catch JSON-related errors and raise as requests.JSONDecodeError\u001b[39;00m\n\u001b[1;32m 974\u001b[0m \u001b[39m# This aliases json.JSONDecodeError and simplejson.JSONDecodeError\u001b[39;00m\n\u001b[0;32m--> 975\u001b[0m \u001b[39mraise\u001b[39;00m RequestsJSONDecodeError(e\u001b[39m.\u001b[39mmsg, e\u001b[39m.\u001b[39mdoc, e\u001b[39m.\u001b[39mpos)\n",
"\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mPaginationError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[19], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39m# settlements_df = sg.query_df(settlements_qp, pagination_strategy=ShallowStrategy) # note both Shallow and Legacy strategies don't work\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m settlements_df \u001b[39m=\u001b[39m sg\u001b[39m.\u001b[39;49mquery_df(settlements_qp)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:380\u001b[0m, in \u001b[0;36mSubgrounds.query_df\u001b[0;34m(self, fpaths, columns, concat, pagination_strategy)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Same as :func:`Subgrounds.query` but formats the response data into a\u001b[39;00m\n\u001b[1;32m 310\u001b[0m \u001b[39mPandas DataFrame. If the response data cannot be flattened to a single query\u001b[39;00m\n\u001b[1;32m 311\u001b[0m \u001b[39m(e.g.: when querying multiple list fields that return different entities),\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 372\u001b[0m \u001b[39m 9 1643213196 2610.686563\u001b[39;00m\n\u001b[1;32m 373\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 374\u001b[0m fpaths \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(\n\u001b[1;32m 375\u001b[0m [fpaths]\n\u001b[1;32m 376\u001b[0m \u001b[39m|\u001b[39m traverse\n\u001b[1;32m 377\u001b[0m \u001b[39m|\u001b[39m \u001b[39mmap\u001b[39m(FieldPath\u001b[39m.\u001b[39m_auto_select)\n\u001b[1;32m 378\u001b[0m \u001b[39m|\u001b[39m traverse\n\u001b[1;32m 379\u001b[0m )\n\u001b[0;32m--> 380\u001b[0m json_data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mquery_json(fpaths, pagination_strategy\u001b[39m=\u001b[39;49mpagination_strategy)\n\u001b[1;32m 381\u001b[0m \u001b[39mreturn\u001b[39;00m df_of_json(json_data, fpaths, columns, concat)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:274\u001b[0m, in \u001b[0;36mSubgrounds.query_json\u001b[0;34m(self, fpaths, pagination_strategy)\u001b[0m\n\u001b[1;32m 267\u001b[0m fpaths \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(\n\u001b[1;32m 268\u001b[0m [fpaths]\n\u001b[1;32m 269\u001b[0m \u001b[39m|\u001b[39m traverse\n\u001b[1;32m 270\u001b[0m \u001b[39m|\u001b[39m \u001b[39mmap\u001b[39m(FieldPath\u001b[39m.\u001b[39m_auto_select)\n\u001b[1;32m 271\u001b[0m \u001b[39m|\u001b[39m traverse\n\u001b[1;32m 272\u001b[0m )\n\u001b[1;32m 273\u001b[0m req \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmk_request(fpaths)\n\u001b[0;32m--> 274\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mexecute(req, pagination_strategy\u001b[39m=\u001b[39;49mpagination_strategy)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:199\u001b[0m, in \u001b[0;36mSubgrounds.execute\u001b[0;34m(self, req, pagination_strategy)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[39mreturn\u001b[39;00m transform\u001b[39m.\u001b[39mtransform_response(req, data)\n\u001b[1;32m 197\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m \u001b[39m# Suppress mypy missing return statement warning\u001b[39;00m\n\u001b[0;32m--> 199\u001b[0m \u001b[39mreturn\u001b[39;00m transform_req(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mglobal_transforms, req)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:191\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_req\u001b[0;34m(transforms, req)\u001b[0m\n\u001b[1;32m 189\u001b[0m \u001b[39mmatch\u001b[39;00m transforms:\n\u001b[1;32m 190\u001b[0m \u001b[39mcase\u001b[39;00m []:\n\u001b[0;32m--> 191\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlist\u001b[39;49m(req\u001b[39m.\u001b[39;49mdocuments \u001b[39m|\u001b[39;49m \u001b[39mmap\u001b[39;49m(\u001b[39mlambda\u001b[39;49;00m doc: transform_doc(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msubgraphs[doc\u001b[39m.\u001b[39;49murl]\u001b[39m.\u001b[39;49m_transforms, doc)))\n\u001b[1;32m 192\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 193\u001b[0m new_req \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_request(req)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:191\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_req.<locals>.<lambda>\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m 189\u001b[0m \u001b[39mmatch\u001b[39;00m transforms:\n\u001b[1;32m 190\u001b[0m \u001b[39mcase\u001b[39;00m []:\n\u001b[0;32m--> 191\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mlist\u001b[39m(req\u001b[39m.\u001b[39mdocuments \u001b[39m|\u001b[39m \u001b[39mmap\u001b[39m(\u001b[39mlambda\u001b[39;00m doc: transform_doc(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msubgraphs[doc\u001b[39m.\u001b[39;49murl]\u001b[39m.\u001b[39;49m_transforms, doc)))\n\u001b[1;32m 192\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 193\u001b[0m new_req \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_request(req)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:183\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_doc\u001b[0;34m(transforms, doc)\u001b[0m\n\u001b[1;32m 181\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 182\u001b[0m new_doc \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_document(doc)\n\u001b[0;32m--> 183\u001b[0m data \u001b[39m=\u001b[39m transform_doc(rest, new_doc)\n\u001b[1;32m 184\u001b[0m \u001b[39mreturn\u001b[39;00m transform\u001b[39m.\u001b[39mtransform_response(doc, data)\n\u001b[1;32m 186\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:183\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_doc\u001b[0;34m(transforms, doc)\u001b[0m\n\u001b[1;32m 181\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 182\u001b[0m new_doc \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_document(doc)\n\u001b[0;32m--> 183\u001b[0m data \u001b[39m=\u001b[39m transform_doc(rest, new_doc)\n\u001b[1;32m 184\u001b[0m \u001b[39mreturn\u001b[39;00m transform\u001b[39m.\u001b[39mtransform_response(doc, data)\n\u001b[1;32m 186\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:180\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.transform_doc\u001b[0;34m(transforms, doc)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[39mmatch\u001b[39;00m transforms:\n\u001b[1;32m 179\u001b[0m \u001b[39mcase\u001b[39;00m []:\n\u001b[0;32m--> 180\u001b[0m \u001b[39mreturn\u001b[39;00m execute_document(doc)\n\u001b[1;32m 181\u001b[0m \u001b[39mcase\u001b[39;00m [transform, \u001b[39m*\u001b[39mrest]:\n\u001b[1;32m 182\u001b[0m new_doc \u001b[39m=\u001b[39m transform\u001b[39m.\u001b[39mtransform_document(doc)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/subgrounds.py:172\u001b[0m, in \u001b[0;36mSubgrounds.execute.<locals>.execute_document\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m 167\u001b[0m subgraph: Subgraph \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39m(\n\u001b[1;32m 168\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msubgraphs\u001b[39m.\u001b[39mvalues()\n\u001b[1;32m 169\u001b[0m \u001b[39m|\u001b[39m where(\u001b[39mlambda\u001b[39;00m sg: sg\u001b[39m.\u001b[39m_url \u001b[39m==\u001b[39m doc\u001b[39m.\u001b[39murl)\n\u001b[1;32m 170\u001b[0m )\n\u001b[1;32m 171\u001b[0m \u001b[39mif\u001b[39;00m pagination_strategy \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m subgraph\u001b[39m.\u001b[39m_is_subgraph:\n\u001b[0;32m--> 172\u001b[0m \u001b[39mreturn\u001b[39;00m paginate(subgraph\u001b[39m.\u001b[39;49m_schema, doc, pagination_strategy\u001b[39m=\u001b[39;49mpagination_strategy)\n\u001b[1;32m 173\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 174\u001b[0m \u001b[39mreturn\u001b[39;00m client\u001b[39m.\u001b[39mquery(doc\u001b[39m.\u001b[39murl, doc\u001b[39m.\u001b[39mgraphql, variables\u001b[39m=\u001b[39mdoc\u001b[39m.\u001b[39mvariables)\n",
"File \u001b[0;32m~/Documents/github/DataStreams/ds/lib/python3.10/site-packages/subgrounds/pagination/pagination.py:89\u001b[0m, in \u001b[0;36mpaginate\u001b[0;34m(schema, doc, pagination_strategy)\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 88\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exn:\n\u001b[0;32m---> 89\u001b[0m \u001b[39mraise\u001b[39;00m PaginationError(exn\u001b[39m.\u001b[39margs[\u001b[39m0\u001b[39m], strategy)\n\u001b[1;32m 91\u001b[0m \u001b[39mreturn\u001b[39;00m data\n\u001b[1;32m 93\u001b[0m \u001b[39mexcept\u001b[39;00m SkipPagination:\n",
"\u001b[0;31mPaginationError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
]
}
],
"source": [
"# settlements_df = sg.query_df(settlements_qp, pagination_strategy=ShallowStrategy) # note both Shallow and Legacy strategies don't work\n",
"settlements_df = sg.query_df(settlements_qp)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"settlements_df.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"settlements_df.head(5)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "ds",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment