Skip to content

Instantly share code, notes, and snippets.

@wesslen

wesslen/anthropic-claude-3-5-encoding-experiments.ipynb

Created June 26, 2024 19:20
Show Gist options
  • Save wesslen/c441bf27c56afd90871452721a844c78 to your computer and use it in GitHub Desktop.
Save wesslen/c441bf27c56afd90871452721a844c78 to your computer and use it in GitHub Desktop.
Download ZIP
anthropic-claude-3-5-encoding-experiments.ipynb
Display the source blob
Display the rendered blob
Raw
anthropic-claude-3-5-encoding-experiments.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"authorship_tag": "ABX9TyOtas3vF9Mo7FZoGjmz4+NM",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/wesslen/c441bf27c56afd90871452721a844c78/anthropic-claude-3-5-encoding-experiments.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"import requests\n",
"import json\n",
"\n",
"# Set your API key\n",
"API_KEY = userdata.get('ANTHROPIC_API_KEY')\n",
"\n",
"# API endpoint\n",
"API_URL = \"https://api.anthropic.com/v1/messages\"\n",
"\n",
"# Headers\n",
"headers = {\n",
" \"Content-Type\": \"application/json\",\n",
" \"X-API-Key\": API_KEY,\n",
" \"anthropic-version\": \"2023-06-01\" # Add the required version header\n",
"}\n",
"\n",
"def generate_text(prompt):\n",
" data = {\n",
" \"model\": \"claude-3-sonnet-20240229\",\n",
" \"max_tokens\": 300,\n",
" \"messages\": [\n",
" {\"role\": \"user\", \"content\": prompt}\n",
" ]\n",
" }\n",
"\n",
" response = requests.post(API_URL, headers=headers, json=data)\n",
"\n",
" print(f\"Status Code: {response.status_code}\")\n",
" print(f\"Response: {response.text}\")\n",
"\n",
" if response.status_code == 200:\n",
" return response.json()[\"content\"][0][\"text\"]\n",
" else:\n",
" return f\"Error: {response.status_code} - {response.text}\"\n",
"\n",
"if __name__ == \"__main__\":\n",
" if not API_KEY:\n",
" print(\"Please set your ANTHROPIC_API_KEY environment variable.\")\n",
" else:\n",
" prompt = \"Tell me a short joke.\"\n",
" result = generate_text(prompt)\n",
" print(result)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "DG28sARk7LGv",
"outputId": "e7de7386-8b41-4f71-f0fc-68cc6fde7048"
},
"execution_count": 7,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Status Code: 200\n",
"Response: {\"id\":\"msg_01MiizRTH83xcfz2gkH69bjG\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-3-sonnet-20240229\",\"content\":[{\"type\":\"text\",\"text\":\"Why can't a bicycle stand up by itself? It's two-tired!\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":13,\"output_tokens\":19}}\n",
"Why can't a bicycle stand up by itself? It's two-tired!\n"
]
}
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2N-sQ2qd3n4y",
"outputId": "cb41b82f-750d-443d-ad24-9ab7ab511982"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"Testing scenario: Code Generation\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function to calculate the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"def f...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Sure, here's a Python function that calculates the Fibonacci sequence up to `n` terms:\n",
"\n",
"```python\n",
"de...\n",
"\n",
"Testing scenario: Factual Generation\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants convert light energy from the sun into chemical energy...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Photosynthesis is the process by which plants convert light energy from the sun into chemical energy...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to produce oxy...\n",
"\n",
"Testing scenario: Creative Generation\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.0, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was the year 2265, ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 0.5, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was the year 2265, ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was the year 2265, ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.1, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was the year 2265, ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 0.5, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 1}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was the year 2265, ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 10}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"It was supposed to be ...\n",
"\n",
"Testing parameters: {'temperature': 1.0, 'top_p': 1.0, 'top_k': 50}\n",
"Response preview: Here's a short story about a time traveler who accidentally changes history:\n",
"\n",
"The Time Traveler's Mi...\n",
"\n",
"Results saved to claude_api_test_results.json\n"
]
}
],
"source": [
"import os\n",
"import requests\n",
"import json\n",
"from itertools import product\n",
"\n",
"from google.colab import userdata\n",
"\n",
"# Set your API key since Colab\n",
"API_KEY = userdata.get('ANTHROPIC_API_KEY')\n",
"\n",
"# API endpoint\n",
"API_URL = \"https://api.anthropic.com/v1/messages\"\n",
"\n",
"# Headers\n",
"headers = {\n",
" \"Content-Type\": \"application/json\",\n",
" \"X-API-Key\": API_KEY,\n",
" \"anthropic-version\": \"2023-06-01\"\n",
"}\n",
"\n",
"# Test scenarios\n",
"scenarios = [\n",
" {\n",
" \"name\": \"Code Generation\",\n",
" \"prompt\": \"Write a Python function to calculate the Fibonacci sequence up to n terms.\"\n",
" },\n",
" {\n",
" \"name\": \"Factual Generation\",\n",
" \"prompt\": \"Explain the process of photosynthesis in plants.\"\n",
" },\n",
" {\n",
" \"name\": \"Creative Generation\",\n",
" \"prompt\": \"Tell me a short story about a time traveler who accidentally changes history.\"\n",
" }\n",
"]\n",
"\n",
"# Parameter ranges to test\n",
"param_ranges = {\n",
" \"temperature\": [0.0, 0.5, 1.0],\n",
" \"top_p\": [0.1, 0.5, 1.0],\n",
" \"top_k\": [1, 10, 50]\n",
"}\n",
"\n",
"def generate_text(prompt, **params):\n",
" data = {\n",
" \"model\": \"claude-3-sonnet-20240229\",\n",
" \"max_tokens\": 300,\n",
" \"messages\": [\n",
" {\"role\": \"user\", \"content\": prompt}\n",
" ],\n",
" **params\n",
" }\n",
"\n",
" response = requests.post(API_URL, headers=headers, json=data)\n",
"\n",
" if response.status_code == 200:\n",
" return response.json()[\"content\"][0][\"text\"]\n",
" else:\n",
" return f\"Error: {response.status_code} - {response.text}\"\n",
"\n",
"def test_parameters():\n",
" results = []\n",
"\n",
" for scenario in scenarios:\n",
" print(f\"\\nTesting scenario: {scenario['name']}\")\n",
"\n",
" for params in product(*param_ranges.values()):\n",
" param_dict = dict(zip(param_ranges.keys(), params))\n",
"\n",
" print(f\"\\nTesting parameters: {param_dict}\")\n",
"\n",
" response = generate_text(scenario['prompt'], **param_dict)\n",
"\n",
" results.append({\n",
" \"scenario\": scenario['name'],\n",
" \"parameters\": param_dict,\n",
" \"response\": response\n",
" })\n",
"\n",
" print(f\"Response preview: {response[:100]}...\")\n",
"\n",
" return results\n",
"\n",
"def save_results(results, filename=\"claude_api_test_results.json\"):\n",
" with open(filename, \"w\") as f:\n",
" json.dump(results, f, indent=2)\n",
" print(f\"\\nResults saved to {filename}\")\n",
"\n",
"if __name__ == \"__main__\":\n",
" if not API_KEY:\n",
" print(\"Please set your ANTHROPIC_API_KEY environment variable.\")\n",
" else:\n",
" results = test_parameters()\n",
" save_results(results)"
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment