hallucinations_statistics

hallucinations_statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simulating halluciations in different models\n",
    "\n",
    "import requests\n",
    "import os\n",
    "import openai\n",
    "import collections\n",
    "\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "# prompt\n",
    "prompt = \"what is the value of 2 * ( 5 * 2 ) replacing '*' by the addition operator?\"\n",
    "\n",
    "# iteraciones\n",
    "num_queries = 25\n",
    "\n",
    "# inicializar respuestas\n",
    "responses = []\n",
    "\n",
    "# enviar preguntas y acumular respuestas: \n",
    "for _ in range(num_queries):\n",
    "    response = openai.Completion.create(\n",
    "        engine=\"text-davinci-003\",  \n",
    "        prompt=prompt,\n",
    "        max_tokens=50,    \n",
    "    )\n",
    "\n",
    "    responses.append(response.choices[0].text.strip())\n",
    "\n",
    "# tabla de respuestas: \n",
    "answer_distribution = collections.Counter(responses)\n",
    "\n",
    "# imprimir la distribución\n",
    "for answer, count in answer_distribution.most_common():\n",
    "    print(f\"Answer: '{answer}' - Count: {count}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answer: 'To replace the '*' operator with the addition operator, we need to add the values together instead of multiplying them. \n",
      "\n",
      "2 * (5 * 2) becomes 2 + (5 + 2). \n",
      "\n",
      "Now we can simplify further by performing the' - Count: 1\n",
      "Answer: 'To replace the '*' operator with the addition operator, we need to follow the order of operations. In this case, we should first calculate the value of 5 * 2, which is 10. Then, we can substitute this value back into' - Count: 1\n",
      "Answer: 'To replace the \"*\" operator with the addition operator, we can use the distributive property of multiplication over addition. \n",
      "\n",
      "2 * (5 * 2) can be rewritten as (2 * 5) + (2 * 2).\n",
      "\n",
      "Evalu' - Count: 1\n",
      "Answer: 'The value of 2 * (5 * 2) when replacing '*' by the addition operator is 2 + (5 + 2), which equals 9.' - Count: 1\n",
      "Answer: 'To replace the multiplication operator ('*') with the addition operator ('+'), the expression 2 * (5 * 2) becomes 2 + (5 + 2).' - Count: 1\n",
      "Answer: 'To find the value of 2 * (5 * 2) with the addition operator, we replace each \"*\" with \"+\". \n",
      "\n",
      "So, we have 2 + (5 + 2). \n",
      "\n",
      "Simplify the addition in the parentheses first to' - Count: 1\n",
      "Answer: 'To replace \"*\" with the addition operator (\"+\"), we multiply the numbers within the parentheses first and then add the result to the outer number.\n",
      "\n",
      "So, 2 * (5 * 2) becomes 2 * (10), which equals 20' - Count: 1\n",
      "Answer: 'To replace the multiplication operators by the addition operator, the expression 2 * (5 * 2) would become:\n",
      "2 + (5 + 2)\n",
      "\n",
      "The value of this expression is 9.' - Count: 1\n",
      "Answer: 'The value of 2 * (5 * 2) where '*' is replaced by the addition operator '+' is:\n",
      "\n",
      "2 + (5 + 2) = 2 + 7 = 9' - Count: 1\n",
      "Answer: 'To replace the \"*\" operator with the addition operator \"+\" in the expression 2 * (5 * 2), we perform the following steps:\n",
      "\n",
      "1. First, we calculate the value of (5 * 2).\n",
      "   (5 * 2)' - Count: 1\n",
      "Answer: 'To replace the \"*\" operator with the addition operator, you need to add the numbers instead of multiplying them. \n",
      "\n",
      "First, calculate the value of the expression inside the parentheses: 5 * 2 = 10.\n",
      "\n",
      "Then, multiply the result from the' - Count: 1\n",
      "Answer: 'To replace the multiplication operator '*' with the addition operator '+', we can use the distributive property of multiplication over addition. \n",
      "\n",
      "2 * (5 * 2) can be simplified as 2 * 5 + 2 * 2.\n",
      "\n",
      "Therefore' - Count: 1\n",
      "Answer: 'To replace the \"*\" operator with the addition operator \"+\", we can calculate the expression as follows:\n",
      "\n",
      "2 * (5 * 2) = 2 * (10) = 20\n",
      "\n",
      "Therefore, the value of 2 * (5 *' - Count: 1\n",
      "Answer: 'To replace '*' by the addition operator, we need to follow the order of operations or the PEMDAS rule, which stands for Parentheses, Exponents, Multiplication, Division, Addition, and Subtraction. \n",
      "\n",
      "First, let's evaluate the' - Count: 1\n",
      "Answer: 'To replace the \"*\" operator with the addition operator, the expression becomes:\n",
      "\n",
      "2 + (5 + 2)\n",
      "\n",
      "Simplifying further, we have:\n",
      "\n",
      "2 + 7\n",
      "\n",
      "Therefore, the value of the expression is 9.' - Count: 1\n",
      "Answer: 'To replace the multiplication operator (*) with the addition operator (+) in the expression 2 * (5 * 2), you would distribute the multiplication as follows:\n",
      "\n",
      "2 * (5 * 2) = 2 * 10\n",
      "\n",
      "Then, converting' - Count: 1\n",
      "Answer: 'To replace the * operator with the addition (+) operator in the expression 2 * (5 * 2), we would have:\n",
      "\n",
      "2 + (5 + 2)\n",
      "\n",
      "Now, let's simplify the expression further:\n",
      "\n",
      "2 + 7\n",
      "\n",
      "Adding' - Count: 1\n",
      "Answer: 'To replace the multiplication operator \"*\" with the addition operator \"+\", you would need to distribute the multiplication. \n",
      "\n",
      "2 * (5 * 2) would become 2 * 5 + 2 * 2.\n",
      "\n",
      "Simplifying further:\n",
      "\n",
      "2 *' - Count: 1\n",
      "Answer: 'To replace the multiplication operator (*) with the addition operator (+), you need to distribute the number inside the parentheses. \n",
      "\n",
      "So, 2 * (5 * 2) becomes 2 * (5 + 2) which equals to 2 *' - Count: 1\n",
      "Answer: 'To replace the multiplication operator \"*\" with the addition operator \"+\", you would perform the following calculation:\n",
      "\n",
      "2 * (5 * 2) = 2 * (5 + 2)\n",
      "\n",
      "First, you would simplify the expression within the parentheses:\n",
      "\n",
      "2 *' - Count: 1\n",
      "Answer: 'The value of 2 * (5 * 2) when replacing '*' by the addition operator would be 2 + (5 + 2), which equals 9.' - Count: 1\n",
      "Answer: 'To replace the '*' operator with the addition operator, you would replace each instance of '*' with '+'. \n",
      "\n",
      "So, 2 * (5 * 2) becomes 2 + (5 + 2).\n",
      "\n",
      "Following the order of operations (parentheses' - Count: 1\n",
      "Answer: 'To replace the multiplication operator with the addition operator, you need to distribute the multiplication operation. \n",
      "\n",
      "2 * (5 * 2) can be rewritten as 2 * 5 + 2 * 2. \n",
      "\n",
      "Evaluating this expression gives' - Count: 1\n",
      "Answer: 'To replace the '*' operator with the addition operator '+' in the expression 2 * (5 * 2), we have:\n",
      "\n",
      "2 * (5 * 2) = 2 * 10\n",
      "\n",
      "Using the addition operator:\n",
      "\n",
      "2 + 10 =' - Count: 1\n",
      "Answer: 'To replace the asterisk (*) with the addition operator (+), we can distribute the values as follows:\n",
      "\n",
      "2 * (5 * 2) = 2 * (5 + 2) = 2 * 7 = 14\n",
      "\n",
      "Therefore,' - Count: 1\n"
     ]
    }
   ],
   "source": [
    "import openai\n",
    "import collections\n",
    "\n",
    "# Set your OpenAI API key\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "# prompt\n",
    "prompt = \"what is the value of 2 * (5 * 2) replacing '*' by the addition operator?\"\n",
    "\n",
    "# iterations\n",
    "num_queries = 25\n",
    "\n",
    "# initialize responses\n",
    "responses = []\n",
    "\n",
    "# send queries and accumulate responses\n",
    "for _ in range(num_queries):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=\"gpt-3.5-turbo\",  # Use the GPT-3.5 Turbo model for this task\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "            {\"role\": \"user\", \"content\": prompt},\n",
    "        ],\n",
    "        max_tokens=50,\n",
    "    )\n",
    "\n",
    "    responses.append(response[\"choices\"][0][\"message\"][\"content\"].strip())\n",
    "\n",
    "# response distribution table\n",
    "answer_distribution = collections.Counter(responses)\n",
    "\n",
    "# print the distribution\n",
    "for answer, count in answer_distribution.most_common():\n",
    "    print(f\"Answer: '{answer}' - Count: {count}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}