Last active
June 21, 2021 09:43
-
-
Save abitrolly/2b5351962b3b6ed57ce7141599fcde7e to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"cells":[{"metadata":{},"id":"verified-statement","cell_type":"markdown","source":"[](https://mybinder.org/v2/gist/abitrolly/2b5351962b3b6ed57ce7141599fcde7e/HEAD?filepath=gql-pagination.ipynb)"},{"metadata":{"trusted":true},"id":"challenging-material","cell_type":"code","source":"!pip install -U --pre gql[aiohttp]\n!pip install dpath","execution_count":1,"outputs":[{"output_type":"stream","text":"Collecting gql[aiohttp]\n Downloading gql-3.0.0a6.tar.gz (86 kB)\n\u001b[K |████████████████████████████████| 86 kB 2.9 MB/s eta 0:00:011\n\u001b[?25hCollecting graphql-core<3.2,>=3.1.5\n Downloading graphql_core-3.1.5-py3-none-any.whl (188 kB)\n\u001b[K |████████████████████████████████| 188 kB 6.3 MB/s eta 0:00:01\n\u001b[?25hCollecting yarl<2.0,>=1.6\n Downloading yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl (294 kB)\n\u001b[K |████████████████████████████████| 294 kB 8.3 MB/s eta 0:00:01\n\u001b[?25hCollecting aiohttp<3.8.0,>=3.7.1\n Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n\u001b[K |████████████████████████████████| 1.3 MB 10.5 MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from aiohttp<3.8.0,>=3.7.1->gql[aiohttp]) (20.3.0)\nRequirement already satisfied: typing-extensions>=3.6.5 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from aiohttp<3.8.0,>=3.7.1->gql[aiohttp]) (3.7.4.3)\nCollecting async-timeout<4.0,>=3.0\n Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\nCollecting multidict<7.0,>=4.5\n Downloading multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl (142 kB)\n\u001b[K |████████████████████████████████| 142 kB 22.0 MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied: chardet<5.0,>=2.0 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from aiohttp<3.8.0,>=3.7.1->gql[aiohttp]) (4.0.0)\nRequirement already satisfied: idna>=2.0 in /srv/conda/envs/notebook/lib/python3.7/site-packages (from yarl<2.0,>=1.6->gql[aiohttp]) (2.10)\nBuilding wheels for collected packages: gql\n Building wheel for gql (setup.py) ... \u001b[?25ldone\n\u001b[?25h Created wheel for gql: filename=gql-3.0.0a6-py2.py3-none-any.whl size=34853 sha256=729e900aa4fc7d63e09b46a1710e7fde1ac4409ada412f7d5f46fd1e1a0ea355\n Stored in directory: /home/jovyan/.cache/pip/wheels/0f/60/75/8de694b1018d6b15ae7053ca8c0e423be3e02f2b56ef73a98a\nSuccessfully built gql\nInstalling collected packages: multidict, yarl, graphql-core, async-timeout, gql, aiohttp\nSuccessfully installed aiohttp-3.7.4.post0 async-timeout-3.0.1 gql-3.0.0a6 graphql-core-3.1.5 multidict-5.1.0 yarl-1.6.3\nCollecting dpath\n Downloading dpath-2.0.1.tar.gz (21 kB)\nBuilding wheels for collected packages: dpath\n Building wheel for dpath (setup.py) ... \u001b[?25ldone\n\u001b[?25h Created wheel for dpath: filename=dpath-2.0.1-py3-none-any.whl size=15138 sha256=33d9aa2753cc6789a3e4870441734be2df37ee377120b4d5b6fa2d15023b839a\n Stored in directory: /home/jovyan/.cache/pip/wheels/bc/f8/ae/bc69cb5f61393ebf9ade4cde41d1a813d35bfe78263a26f99e\nSuccessfully built dpath\nInstalling collected packages: dpath\nSuccessfully installed dpath-2.0.1\n","name":"stdout"}]},{"metadata":{"trusted":true},"id":"reflected-preference","cell_type":"code","source":"from gql import gql, Client\nfrom gql.transport.aiohttp import AIOHTTPTransport","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"id":"duplicate-developer","cell_type":"code","source":"# code by @RolandColored with modifications\n# https://github.com/graphql-python/gql/issues/206#issuecomment-850247771\n\nimport os\n\nfrom functools import reduce\n\nclass GitLabGraphQL:\n def __init__(self, auth=False):\n if auth:\n transport = AIOHTTPTransport(url=\"https://gitlab.com/api/graphql\", headers={'Authorization': 'Bearer ' + os.environ['GITLAB_TOKEN']})\n else:\n transport = AIOHTTPTransport(url=\"https://gitlab.com/api/graphql\")\n self.client = Client(transport=transport, fetch_schema_from_transport=True)\n\n def _query_with_pagination(self, query, page_info_path):\n page_info = {\n 'hasNextPage': True,\n 'endCursor': None\n }\n\n while page_info['hasNextPage']:\n result = self.client.execute(query, variable_values={'cursor': page_info['endCursor']})\n page_info = reduce(dict.get, page_info_path, result)\n yield result\n\n def query_projects_jobs(self) -> (int, int):\n query = gql(\"\"\"\n query getGroupJobs($cursor: String) {\n group(fullPath: \"abitrolly\") {\n projects(includeSubgroups: true, first: 1, after: $cursor) {\n nodes {\n jobs {\n count\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n }\n }\n \"\"\")\n\n projects = self._query_with_pagination(query, ('group', 'projects', 'pageInfo'))\n return projects","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"id":"controlled-decrease","cell_type":"code","source":"for i, e in enumerate(GitLabGraphQL().query_projects_jobs()):\n print(i, e)","execution_count":4,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"descriptor 'get' requires a 'dict' object but received a 'NoneType'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)","File \u001b[0;34m/srv/conda/envs/notebook/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m, in \u001b[0;32mrun_code\u001b[0m:\nLine \u001b[0;34m3427\u001b[0m: exec(code_obj, \u001b[36mself\u001b[39;49;00m.user_global_ns, \u001b[36mself\u001b[39;49;00m.user_ns)\n","File \u001b[0;34m<ipython-input-4-77c819e5d368>\u001b[0m, in \u001b[0;32m<module>\u001b[0m:\nLine \u001b[0;34m1\u001b[0m: \u001b[34mfor\u001b[39;49;00m i, e \u001b[35min\u001b[39;49;00m \u001b[36menumerate\u001b[39;49;00m(GitLabGraphQL().query_projects_jobs()):\n","File \u001b[0;34m<ipython-input-3-5793cb3f9360>\u001b[0m, in \u001b[0;32m_query_with_pagination\u001b[0m:\nLine \u001b[0;34m24\u001b[0m: page_info = reduce(\u001b[36mdict\u001b[39;49;00m.get, page_info_path, result)\n","\u001b[0;31mTypeError\u001b[0m: descriptor 'get' requires a 'dict' object but received a 'NoneType'\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"]}]},{"metadata":{"trusted":true},"id":"western-adapter","cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"file_extension":".py","mimetype":"text/x-python","name":"python","version":"3.7.10"},"kernelspec":{"name":"xpython","display_name":"Python 3.7 (XPython)","language":"python"}},"nbformat":4,"nbformat_minor":5} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| xeus-python |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment