Last active
May 16, 2023 06:07
-
-
Save qlawmarq/22b488d2a3cd18353d8d05c6c66bb9db to your computer and use it in GitHub Desktop.
Search GitHub Starred repo by LlamaIndex (GPT Index)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "fb0b45cc-4268-4492-9004-34852ecdf651", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"!pip install llama-index==0.6.7\n", | |
"!pip install html2text==2020.1.16" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "d4387cd5-20f5-4fa8-ac8d-73c83dc2d4b6", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"import json\n", | |
"import urllib.request\n", | |
"\n", | |
"from llama_index import download_loader\n", | |
"\n", | |
"urlData = \"https://api.github.com/users/octocat/starred\"\n", | |
"webURL = urllib.request.urlopen(urlData)\n", | |
"data = webURL.read()\n", | |
"encoding = webURL.info().get_content_charset('utf-8')\n", | |
"jsonData = json.loads(data.decode(encoding))\n", | |
"\n" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "2673a85e-d996-45c3-8cd8-b7a59ce54f41", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"from llama_index import GPTVectorStoreIndex, SimpleWebPageReader\n", | |
"\n", | |
"import os\n", | |
"os.environ[\"OPENAI_API_KEY\"] = \"your-openai-api-key-xxx\"\n", | |
"\n", | |
"starred_github_urls = []\n", | |
"\n", | |
"for obj in jsonData:\n", | |
" starred_github_urls.append(obj['html_url'])\n", | |
"\n", | |
"documents = SimpleWebPageReader().load_data(starred_github_urls)\n", | |
"index = GPTVectorStoreIndex.from_documents(documents)\n" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "bf650857-115f-4461-b2f4-8714d28712c8", | |
"metadata": { | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"query_engine = index.as_query_engine()\n", | |
"response = query_engine.query(\"Tell me a GitHub repogitory url that created by octocat\")\n", | |
"print(response)" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.10.11" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment