Created
July 19, 2023 18:22
-
-
Save Cdaprod/fb4315ce0ddd9c61e731fb2ea929a170 to your computer and use it in GitHub Desktop.
Langchain scripts for Dataloader and ConversationalChain
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "f4d521f0", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from langchain.document_loaders import TextLoader\n", | |
"from langchain.text_splitter import CharacterTextSplitter\n", | |
"import os\n", | |
"\n", | |
"# Load and split your source code and documentation\n", | |
"root_dir = '/directory'\n", | |
"docs = []\n", | |
"for dirpath, dirnames, filenames in os.walk(root_dir):\n", | |
" for file in filenames:\n", | |
" if file.endswith('.py') and '/.venv/' not in dirpath:\n", | |
" try:\n", | |
" loader = TextLoader(os.path.join(dirpath, file), encoding='utf-8')\n", | |
" docs.extend(loader.load_and_split())\n", | |
" except Exception as e:\n", | |
" pass\n", | |
"\n", | |
"text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n", | |
"texts = text_splitter.split_documents(docs)\n", | |
"\n" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"id": "e4e250f2", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"# Search for relevant information using LangChain's conversational retrieval chain\n", | |
"from langchain.chains import ConversationalRetrievalChain\n", | |
"from langchain.chat_models import ChatOpenAI\n", | |
"\n", | |
"chain = ConversationalRetrievalChain(ChatOpenAI())\n", | |
"questions = ['How do I write to a file in Python?']\n", | |
"answers = chain.run(questions, texts)\n", | |
"\n", | |
"# Write the relevant information to a file\n", | |
"with open('output.txt', 'w') as f:\n", | |
" f.write(answers[0])" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.11.2" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment