01_dataload_pg_lamaIndex.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c5289586-0778-4203-a302-1782e8d4c173",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !pip install llama-index\n",
    "# !pip install pgvector asyncpg psycopg2-binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a85f7cfc-836b-4b4e-8b23-4fd95884a26b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Model model id=TheBloke_WizardCoder-Python-34B-V1.0-GPTQ at 0x7f85c8883b00> JSON: {\n",
       "  \"id\": \"TheBloke_WizardCoder-Python-34B-V1.0-GPTQ\",\n",
       "  \"object\": \"model\",\n",
       "  \"owned_by\": \"user\",\n",
       "  \"permission\": []\n",
       "}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.0 Set local openai before any imports\n",
    "import os\n",
    "os.environ['OPENAI_API_KEY']=\"sk-111111111111111111111111111111111111111111111111\"\n",
    "os.environ['OPENAI_API_BASE']=\"http://playground:5001/v1\"\n",
    "import openai\n",
    "# 1.1 Check the remote model being used\n",
    "openai.Model.list()['data'][0]#['id']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "14f30416-2287-4581-8263-50967fe57efb",
   "metadata": {},
   "source": [
    "## llama index + Pg Vector - 01.Data load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a35eb73c-800d-43c9-84f7-8eb4d861721e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index import SimpleDirectoryReader, StorageContext\n",
    "from llama_index.indices.vector_store import VectorStoreIndex\n",
    "from llama_index.vector_stores import PGVectorStore\n",
    "import textwrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "987c83ed-662e-4936-a00d-e834b145346b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# SimpleDirectoryReader?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0dc304f5-28b7-41c0-b039-bca541a29a8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 7394ed9e-f21e-461e-99c6-6a0ac309667c\n"
     ]
    }
   ],
   "source": [
    "documents = SimpleDirectoryReader(input_files=[\"/data/nvme1/workspace/dev/pg_vector_dev/pihole.5.txt\"]).load_data()\n",
    "print(\"Document ID:\", documents[0].doc_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "46af9d28-1c12-4129-8781-477f660e0e68",
   "metadata": {},
   "outputs": [],
   "source": [
    "# SimpleDirectoryReader?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d373827f-ac2b-4817-a106-40eb5024cacf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import psycopg2\n",
    "connection_string ='postgresql://x:x@playground:5432/test'\n",
    "db_name = \"test\"\n",
    "conn = psycopg2.connect(connection_string)\n",
    "conn.autocommit = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a1f3aae1-28db-441d-9831-18da063b8b6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# VectorStoreIndex.from_documents?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "489abd71-bc48-47dd-9101-95f31454e740",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parsing documents into nodes: 100%|██████████| 1/1 [04:23<00:00, 263.02s/it]\n",
      "Generating embeddings: 100%|██████████| 14598/14598 [51:36<00:00,  4.71it/s] \n"
     ]
    }
   ],
   "source": [
    "from sqlalchemy import make_url\n",
    "url = make_url(connection_string)\n",
    "vector_store = PGVectorStore.from_params(\n",
    "    database=db_name,\n",
    "    host=url.host,\n",
    "    password=url.password,\n",
    "    port=url.port,\n",
    "    user=url.username,\n",
    "    table_name=\"pihole_1_test_3\",\n",
    "    embed_dim=768,   \n",
    ")\n",
    "\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    documents, storage_context=storage_context, show_progress=True\n",
    ")\n",
    "query_engine = index.as_query_engine()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (wizmath)",
   "language": "python",
   "name": "conda-env-wizmath-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}