exllama+sentence transformers+qdrant

vectorize_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set relevant parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOOK_FILENAME = \"Marcus_Aurelius_Antoninus_-_His_Meditations_concerning_himselfe\"\n",
    "DATA = \"./data\"\n",
    "COLLECTION_NAME = \"dip\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Connect to Qdrant and create collection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from qdrant_client import QdrantClient\n",
    "from qdrant_client.http import models\n",
    "\n",
    "client = QdrantClient(\":memory:\")\n",
    "client.recreate_collection(\n",
    "    collection_name=COLLECTION_NAME,\n",
    "    vectors_config=models.VectorParams(size=384, distance=models.Distance.COSINE),\n",
    ")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read sentences\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3a62267e7924a2aabe5036811b80003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with open(f\"{DATA}/processed/{BOOK_FILENAME}/{BOOK_FILENAME}.json\", \"r\") as file:\n",
    "    meditations_json = json.load(file)\n",
    "\n",
    "rows = []\n",
    "for chapter in tqdm(meditations_json[\"data\"]):\n",
    "    for sentence in chapter[\"sentences\"]:\n",
    "        rows.append(\n",
    "            (\n",
    "                chapter[\"title\"],\n",
    "                chapter[\"url\"],\n",
    "                sentence,\n",
    "            )\n",
    "        )\n",
    "\n",
    "df = pd.DataFrame(data=rows, columns=[\"title\", \"url\", \"sentence\"])\n",
    "\n",
    "df = df[df[\"sentence\"].str.split().str.len() > 15]\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorize sentences\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fed4729d27545959aadbc561c0a56d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/634 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "UpdateResult(operation_id=0, status=<UpdateStatus.COMPLETED: 'completed'>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "sentence_model = SentenceTransformer(\n",
    "    \"sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2\",\n",
    "    device=\"cuda\"\n",
    "    if torch.cuda.is_available()\n",
    "    else \"mps\"\n",
    "    if torch.backends.mps.is_available()\n",
    "    else \"cpu\",\n",
    ")\n",
    "\n",
    "vectors = []\n",
    "batch_size = 512\n",
    "batch = []\n",
    "\n",
    "for doc in tqdm(df[\"sentence\"].to_list()):\n",
    "    batch.append(doc)\n",
    "\n",
    "    if len(batch) >= batch_size:\n",
    "        vectors.append(sentence_model.encode(batch))\n",
    "        batch = []\n",
    "\n",
    "if len(batch) > 0:\n",
    "    vectors.append(sentence_model.encode(batch))\n",
    "    batch = []\n",
    "\n",
    "vectors = np.concatenate(vectors)\n",
    "\n",
    "book_name = meditations_json[\"book_title\"]\n",
    "\n",
    "client.upsert(\n",
    "    collection_name=COLLECTION_NAME,\n",
    "    points=models.Batch(\n",
    "        ids=[i for i in range(df.shape[0])],\n",
    "        payloads=[\n",
    "            {\n",
    "                \"text\": row[\"sentence\"],\n",
    "                \"title\": row[\"title\"] + f\", {book_name}\",\n",
    "                \"url\": row[\"url\"],\n",
    "            }\n",
    "            for _, row in df.iterrows()\n",
    "        ],\n",
    "        vectors=[v.tolist() for v in vectors],\n",
    "    ),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "def log(message):\n",
    "    timestamp = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    print(f\"[{timestamp}] {message}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-08-31 00:12:52] >>> model loading...\n",
      "[2023-08-31 00:12:54] >>> model loaded...\n"
     ]
    }
   ],
   "source": [
    "from exllama.model import ExLlama, ExLlamaCache, ExLlamaConfig\n",
    "from exllama.lora import ExLlamaLora\n",
    "from exllama.tokenizer import ExLlamaTokenizer\n",
    "from exllama.generator import ExLlamaGenerator\n",
    "\n",
    "import torch\n",
    "\n",
    "torch.set_grad_enabled(False)\n",
    "torch.cuda._lazy_init()\n",
    "\n",
    "config_path = \"../models/MythoMix-L2-13B-GPTQ\"\n",
    "config = ExLlamaConfig(f\"{config_path}/config.json\")\n",
    "config.model_path = f\"{config_path}/model.safetensors\"\n",
    "model = ExLlama(config)\n",
    "\n",
    "log(\">>> model loading...\")\n",
    "model = ExLlama(config)\n",
    "log(\">>> model loaded...\")\n",
    "cache = ExLlamaCache(model)\n",
    "tokenizer = ExLlamaTokenizer(f\"{config_path}/tokenizer.model\")\n",
    "\n",
    "generator = ExLlamaGenerator(model, tokenizer, cache)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_prompt(question: str, references: list) -> tuple[str, str]:\n",
    "    prompt = f\"\"\"\n",
    "    You're Marcus Aurelius, emperor of Rome. You're giving advice to a friend who has asked you the following question: '{question}'\n",
    "\n",
    "    You've selected the most relevant passages from your writings to use as source for your answer. Cite them in your answer.\n",
    "    References:\n",
    "    \"\"\".strip()\n",
    "\n",
    "    references_text = \"\"\n",
    "\n",
    "    for i, reference in enumerate(references, start=1):\n",
    "        text = reference.payload[\"text\"].strip()\n",
    "        references_text += f\"\\n[{i}]: {text}\"\n",
    "\n",
    "    prompt += (\n",
    "        references_text\n",
    "        + \"\\nHow to cite a reference: This is a citation [1]. This one too [3]. And this is sentence with many citations [2][3].\\nAnswer:\"\n",
    "    )\n",
    "    return prompt, references_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You're Marcus Aurelius, emperor of Rome. You're giving advice to a friend who has asked you the following question: 'how to be a good person?'\n",
      "\n",
      "    You've selected the most relevant passages from your writings to use as source for your answer. Cite them in your answer.\n",
      "    References:\n",
      "[1]: XXI. Try also how a good man's life; (of one, who is well pleased with those things whatsoever, which among the common changes and chances of this world fall to his own lot and share; and can live well contented and fully satisfied in the justice of his own proper present action, and in the goodness of his disposition for the future:) will agree with thee. Thou hast had experience of that other kind of life: make now trial of this also. Trouble not thyself any more henceforth, reduce thyself unto perfect simplicity.\n",
      "[2]: XVIII. Make it not any longer a matter of dispute or discourse, what are the signs and proprieties of a good man, but really and actually to be such.\n",
      "[3]: XII. What those things are in themselves, which by the greatest part are esteemed good, thou mayest gather even from this. For if a man shall hear things mentioned as good, which are really good indeed, such as are prudence, temperance, justice, fortitude, after so much heard and conceived, he cannot endure to hear of any more, for the word good is properly spoken of them. But as for those which by the vulgar are esteemed good, if he shall hear them mentioned as good, he doth hearken for more.\n",
      "How to cite a reference: This is a citation [1]. This one too [3]. And this is sentence with many citations [2][3].\n",
      "Answer:\n"
     ]
    }
   ],
   "source": [
    "question = \"how to be a good person?\"\n",
    "similar_docs = client.search(\n",
    "    collection_name=COLLECTION_NAME,\n",
    "    query_vector=sentence_model.encode(question),\n",
    "    limit=3,\n",
    "    append_payload=True,\n",
    ")\n",
    "\n",
    "prompt, references = build_prompt(question, similar_docs)\n",
    "\n",
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You're Marcus Aurelius, emperor of Rome. You're giving advice to a friend who has asked you the following question: 'how to be a good person?'\n",
      "\n",
      "    You've selected the most relevant passages from your writings to use as source for your answer. Cite them in your answer.\n",
      "    References:\n",
      "[1]: XXI. Try also how a good man's life; (of one, who is well pleased with those things whatsoever, which among the common changes and chances of this world fall to his own lot and share; and can live well contented and fully satisfied in the justice of his own proper present action, and in the goodness of his disposition for the future:) will agree with thee. Thou hast had experience of that other kind of life: make now trial of this also. Trouble not thyself any more henceforth, reduce thyself unto perfect simplicity.\n",
      "[2]: XVIII. Make it not any longer a matter of dispute or discourse, what are the signs and proprieties of a good man, but really and actually to be such.\n",
      "[3]: XII. What those things are in themselves, which by the greatest part are esteemed good, thou mayest gather even from this. For if a man shall hear things mentioned as good, which are really good indeed, such as are prudence, temperance, justice, fortitude, after so much heard and conceived, he cannot endure to hear of any more, for the word good is properly spoken of them. But as for those which by the vulgar are esteemed good, if he shall hear them mentioned as good, he doth hearken for more.\n",
      "How to cite a reference: This is a citation [1]. This one too [3]. And this is sentence with many citations [2][3].\n",
      "Answer: Marcus Aurelius' advice on how to be a good person focuses on living a simple and virtuous life, characterized by qualities like prudence, temperance, justice, and fortitude. He suggests that true happiness comes from within oneself, rather than external circumstances, and encourages individuals to strive for inner peace and contentment through moral excellence. As stated in passage [1], \"Try also how a good man's life; (of one, who is well pleased with those things whatsoever, which among the common changes and chances of this world fall to his own lot and share; and can live well contented and fully satisfied in the justice of his own proper present action, and in the goodness of his disposition for the future:) will agree with thee.\" Furthermore, in passage [2], he emphasizes the importance of actually embodying these virtues, stating \"Make it not any longer a matter of dispute or discourse, what are the signs and proprieties of a good man, but really and actually to be such.\" Lastly, in passage [3], he provides insight into human nature and our desire for genuine goodness, suggesting that we naturally seek out information about truly good things because they resonate with us at a deeper level.\n"
     ]
    }
   ],
   "source": [
    "response = generator.generate_simple(prompt, max_new_tokens = 500)\n",
    "\n",
    "print(response)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "7f7d7ce7694bb4f4c294d506e5b6dc7957106f5332d820f0757e3d8cd7b1bbf8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}