Notebook demonstrating the way that you ask GPT-3.5 a question changes the response. The function calling API performs the worst.

lease-answering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pet_clause = \"\"\"20. PETS: No pets are permitted on the Property or the Premises without the prior written consent of the Landlord. Any\n",
    "such consent may be revoked at any time, with or without cause, by giving three (3) days written notice. Except to the\n",
    "extent written permission is given, pets may not be brought upon the Property or the Premises, whether such pets belong\n",
    "to Resident or to any other person. The presence of any pets for which written permission has not been given or which, if\n",
    "given, is not currently in force, even if such pets are 'just visiting\", shall be deemed a material breach of this Lease and\n",
    "shall be cause for the service of a three (3) day notice to perform covenants and conditions or quit. If, in accordance with\n",
    "the provisions of this paragraph a pet is permitted, Resident shall (a) pay an increase in the Security Deposit, (b) pay any\n",
    "applicable rent, and (c) sign a pet policy for each pet known as addendum C of this Lease, prior to any pet being allowed in\n",
    "the Premises or on the Property. If Resident has a pet without the written consent of Landlord as documented on such pet\n",
    "policy form for each pet, in addition to all other remedies of Landlord, Resident agrees to pay within three (3) days of\n",
    "written demand therefore any and all carpet cleaning charges, fumigation costs and any and all damages caused by\n",
    "unauthorized pets, even if such charges should exceed the amount of any and all deposits held by Landlord. (See\n",
    "attached Pet Policy Addendum.) Assistive (including companion) animals for the disabled are not considered to be pets,\n",
    "but do require prior written approval of management.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yes\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant that answers questions about of leases with single-word answers.\"},\n",
    "    {\"role\": \"user\", \"content\": 'Given the following clause, would pets be allowed if there is prior written consent? Please answer \"Yes\" or \"No\".\\n\\n' + pet_clause}\n",
    "]\n",
    "\n",
    "for i in range(1):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model='gpt-3.5-turbo-0613',\n",
    "        temperature=0,\n",
    "        messages=messages)\n",
    "    print(response.choices[0]['message']['content'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"pets_allowed\": true}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant that answers questions about of leases with JSON responses.\"},\n",
    "    {\"role\": \"user\", \"content\": 'Given the following clause, would pets be allowed if there is prior written consent? Please answer {\\\"pets_allowed\\\": true} or {\\\"pets_allowed\\\": false}.\\n\\n' + pet_clause}\n",
    "]\n",
    "\n",
    "for i in range(1):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model='gpt-3.5-turbo-0613',\n",
    "        temperature=0,\n",
    "        messages=messages)\n",
    "    print(response.choices[0]['message']['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"pets_allowed\": false}\n",
      "{\"pets_allowed\": true}\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant that answers questions about the following lease clause: \" + pet_clause},\n",
    "    {\"role\": \"user\", \"content\": 'Extract data from the lease clause.'}\n",
    "]\n",
    "\n",
    "functions = [\n",
    "    {\n",
    "        \"name\": \"extract_data\",\n",
    "        \"description\": \"Extract the details of a lease clause.\",\n",
    "        \"parameters\": {\n",
    "            \"type\": \"object\",\n",
    "            \"properties\": {\n",
    "                \"pets_allowed\": {\"type\": \"boolean\", \"description\": \"Would pets be allowed if there is prior written consent?\"},\n",
    "            },\n",
    "            \"required\": [],\n",
    "        }\n",
    "    }\n",
    "]\n",
    "\n",
    "for i in range(2):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model='gpt-3.5-turbo-0613',\n",
    "        temperature=0,\n",
    "        functions=functions,\n",
    "        messages=messages)\n",
    "    response = response.choices[0]['message']['function_call']['arguments']\n",
    "    print(json.dumps(json.loads(response)))\n",
    "\n",
    "    # change the function description very slightly for the second pass\n",
    "    functions[0]['description'] = \"Extract the details of a lease.\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"pets_allowed\": true}\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant that answers questions about the following lease clause: \" + pet_clause},\n",
    "    {\"role\": \"user\", \"content\": 'Extract a summary from the lease clause and add them to the database.'}\n",
    "]\n",
    "\n",
    "functions = [\n",
    "    {\n",
    "        \"name\": \"add_lease_details\",\n",
    "        \"description\": \"Add the details of a lease to the database.\",\n",
    "        \"parameters\": {\n",
    "            \"type\": \"object\",\n",
    "            \"properties\": {\n",
    "                \"pets_allowed\": {\"type\": \"boolean\", \"description\": \"Would pets be allowed if there is prior written consent?\"},\n",
    "            },\n",
    "            \"required\": [],\n",
    "        }\n",
    "    }\n",
    "]\n",
    "\n",
    "for i in range(1):\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model='gpt-3.5-turbo-0613',\n",
    "        temperature=0,\n",
    "        functions=functions,\n",
    "        messages=messages)\n",
    "    response = response.choices[0]['message']['function_call']['arguments']\n",
    "    print(json.dumps(json.loads(response)))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "question-answering",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

Put the pet_clause in the system role and lower temperature to zero. This model is fine-tuned for system messages with FC.