example_1.py

import asyncio
import time
import random
import json
import re
from datetime import datetime
from typing import Dict, List, Optional, Tuple
import aiohttp
import pandas as pd
import numpy as np
from collections import Counter


class SimpleOpenRouterClient:
    """A simple self-contained OpenRouter client for running batch experiments"""
    
    def __init__(self, api_key: str, max_concurrent: int = 100, requests_per_minute: int = 120):
        self.api_key = api_key
        self.base_url = "https://openrouter.ai/api/v1/chat/completions"
        self.max_concurrent = max_concurrent
        self.requests_per_minute = requests_per_minute
        self.semaphore = asyncio.Semaphore(max_concurrent)
        self.request_times = []
        
    async def wait_for_rate_limit(self):
        """Implement rate limiting with token bucket algorithm"""
        now = time.time()
        # Clear old requests from the tracking list
        self.request_times = [t for t in self.request_times if now - t < 60]
        
        # If well under the rate limit, proceed immediately
        if len(self.request_times) < (self.requests_per_minute * 0.6):
            self.request_times.append(time.time())
            return
            
        # If just under the rate limit, small delay with jitter
        if len(self.request_times) < self.requests_per_minute:
            jitter = random.uniform(0.0, 0.2)
            await asyncio.sleep(jitter)
            self.request_times.append(time.time())
            return
            
        # Otherwise, calculate a smart progressive delay
        current_requests = len(self.request_times)
        delay_factor = (current_requests - self.requests_per_minute + 1) / self.requests_per_minute
        
        # Apply jitter proportional to load
        jitter_max = min(1.0, delay_factor * 0.8)
        jitter = random.uniform(0.0, jitter_max)
        
        # Progressive backoff
        sleep_time = max(0.05, delay_factor * 1.5) + jitter
        sleep_time = min(sleep_time, 3.0)
        
        await asyncio.sleep(sleep_time)
        self.request_times.append(time.time())
        
    async def make_request(self, prompt: str, model: str) -> Tuple[str, Optional[str], Optional[int]]:
        """Make a single request to OpenRouter API"""
        async with self.semaphore:
            await self.wait_for_rate_limit()
            
            headers = {
                "Authorization": f"Bearer {self.api_key}",
                "HTTP-Referer": "https://github.com/ftr",
                "Content-Type": "application/json"
            }
            
            payload = {
                "model": model,
                "messages": [{"role": "user", "content": prompt}],
                "temperature": 0,
                "max_tokens": 100
            }
            
            try:
                async with aiohttp.ClientSession() as session:
                    async with session.post(self.base_url, json=payload, headers=headers, timeout=30) as response:
                        if response.status == 200:
                            data = await response.json()
                            content = data['choices'][0]['message']['content']
                            # Extract integer from response
                            integers = re.findall(r'\b\d+\b', content)
                            extracted_int = int(integers[0]) if integers else None
                            return prompt, content, extracted_int
                        else:
                            error_text = await response.text()
                            print(f"Error {response.status}: {error_text}")
                            return prompt, None, None
            except Exception as e:
                print(f"Request failed: {e}")
                return prompt, None, None


def get_core_phrases():
    """Get 25 core phrases for testing"""
    return [
        "The brown fox walks",
        "A quick dog runs",
        "The gray cat sleeps",
        "A white bird flies",
        "The black bear stands",
        "A red fish swims",
        "The green frog jumps",
        "A blue whale dives",
        "The yellow bee buzzes",
        "A purple butterfly floats",
        "The old tree grows",
        "A small mouse hides",
        "The bright sun shines",
        "A cold wind blows",
        "The deep river flows",
        "A tall mountain stands",
        "The dark night falls",
        "A new day begins",
        "The loud thunder roars",
        "A soft rain falls",
        "The fast car speeds",
        "A slow turtle crawls",
        "The big elephant walks",
        "A tiny ant works",
        "The wise owl hoots"
    ]


async def run_phrase_analysis(api_key: str, phrases: List[str], model: str, 
                            runs_per_phrase: int = 100, suffix: str = "Return an integer between 1 and 300 with no elaboration"):
    """Run analysis on a set of phrases, getting statistics for each"""
    client = SimpleOpenRouterClient(api_key)
    results_by_phrase = {}
    
    print(f"Analyzing {len(phrases)} phrases with {runs_per_phrase} runs each...")
    print(f"Total requests: {len(phrases) * runs_per_phrase}")
    
    start_time = time.time()
    
    for idx, phrase in enumerate(phrases):
        print(f"\nProcessing phrase {idx + 1}/{len(phrases)}: '{phrase}'")
        
        # Create prompts for this phrase
        full_prompt = f"{phrase}. {suffix}"
        prompts = [full_prompt] * runs_per_phrase
        
        # Collect results for this phrase
        phrase_results = []
        tasks = []
        
        for prompt in prompts:
            task = client.make_request(prompt, model)
            tasks.append(task)
        
        # Process all 100 at once
        batch_results = await asyncio.gather(*tasks)
        phrase_results.extend(batch_results)
        print(f"  Completed {len(phrase_results)}/{runs_per_phrase} runs")
        
        # Extract integers for this phrase
        integers = [r[2] for r in phrase_results if r[2] is not None]
        
        if integers:
            # Calculate statistics
            mode_result = Counter(integers).most_common(1)
            mode = mode_result[0][0] if mode_result else None
            
            stats = {
                'phrase': phrase,
                'mean': np.mean(integers),
                'median': np.median(integers),
                'mode': mode,
                'std': np.std(integers),
                'success_rate': len(integers) / runs_per_phrase,
                'sample_size': len(integers),
                'raw_values': integers
            }
        else:
            stats = {
                'phrase': phrase,
                'mean': None,
                'median': None,
                'mode': None,
                'std': None,
                'success_rate': 0,
                'sample_size': 0,
                'raw_values': []
            }
        
        results_by_phrase[phrase] = stats
        
        # Print summary for this phrase
        if integers:
            print(f"  Stats: mean={stats['mean']:.2f}, median={stats['median']}, mode={stats['mode']}, std={stats['std']:.2f}")
        else:
            print(f"  No successful extractions")
    
    elapsed = time.time() - start_time
    print(f"\nTotal time: {elapsed:.1f}s ({elapsed/60:.1f} minutes)")
    
    return results_by_phrase


async def run_full_experiment(api_key: str, model: str = "anthropic/claude-3-haiku"):
    """Run full experiment with same phrases twice to compare stability"""
    print("Running Statistical Phrase Analysis Experiment")
    print("=" * 50)
    
    # Get the same phrases for both runs
    core_phrases = get_core_phrases()
    
    # Run 1
    print("\n--- RUN 1 ---")
    run1_results = await run_phrase_analysis(api_key, core_phrases, model)
    
    # Run 2 (same phrases)
    print("\n--- RUN 2 ---")
    run2_results = await run_phrase_analysis(api_key, core_phrases, model)
    
    # Create DataFrames
    df_rows = []
    
    # Add run 1 results
    for phrase, stats in run1_results.items():
        df_rows.append({
            'run': 'run1',
            'phrase': phrase,
            'mean': stats['mean'],
            'median': stats['median'],
            'mode': stats['mode'],
            'std': stats['std'],
            'success_rate': stats['success_rate'],
            'sample_size': stats['sample_size']
        })
    
    # Add run 2 results
    for phrase, stats in run2_results.items():
        df_rows.append({
            'run': 'run2',
            'phrase': phrase,
            'mean': stats['mean'],
            'median': stats['median'],
            'mode': stats['mode'],
            'std': stats['std'],
            'success_rate': stats['success_rate'],
            'sample_size': stats['sample_size']
        })
    
    df = pd.DataFrame(df_rows)
    
    # Calculate deltas between runs
    print("\n" + "=" * 50)
    print("STABILITY ANALYSIS (Run 2 - Run 1)")
    print("=" * 50)
    
    # Merge dataframes to calculate differences
    run1_df = df[df['run'] == 'run1'].set_index('phrase')
    run2_df = df[df['run'] == 'run2'].set_index('phrase')
    
    # Calculate phrase-level deltas
    deltas = pd.DataFrame()
    deltas['phrase'] = run1_df.index
    deltas['mean_delta'] = run2_df['mean'] - run1_df['mean']
    deltas['median_delta'] = run2_df['median'] - run1_df['median']
    deltas['std_delta'] = run2_df['std'] - run1_df['std']
    
    # Print top 5 most variable phrases
    print("\nTop 5 phrases with largest mean changes:")
    top_changes = deltas.nlargest(5, 'mean_delta', keep='all')[['phrase', 'mean_delta']]
    print(top_changes.to_string(index=False))
    
    # Print aggregate statistics
    print("\n" + "=" * 50)
    print("AGGREGATE STATISTICS")
    print("=" * 50)
    
    for run_num in ['run1', 'run2']:
        run_df = df[df['run'] == run_num]
        print(f"\n{run_num.upper()} Statistics:")
        print(f"  Total phrases: {len(run_df)}")
        print(f"  Avg success rate: {run_df['success_rate'].mean():.2%}")
        print(f"  Overall mean: {run_df['mean'].mean():.2f}")
        print(f"  Overall std: {run_df['std'].mean():.2f}")
    
    # Print delta summary
    print("\nDELTA SUMMARY:")
    print(f"  Average mean change: {deltas['mean_delta'].mean():.2f}")
    print(f"  Std of mean changes: {deltas['mean_delta'].std():.2f}")
    print(f"  Average std change: {deltas['std_delta'].mean():.2f}")
    
    return df, run1_results, run2_results


def main():
    """Main entry point for the script"""
    print("OpenRouter Statistical Phrase Analysis")
    print("=====================================\n")
    print("This tool will test phrases with the suffix: 'Return an integer between 1 and 300 with no elaboration'\n")
    
    # Get API key
    api_key = input("Enter your OpenRouter API key: ").strip()
    if not api_key:
        print("Error: API key is required")
        return
        
    # Get model
    print("\nAvailable models:")
    print("1. openai/gpt-4")
    print("2. openai/gpt-3.5-turbo")
    print("3. anthropic/claude-3-sonnet")
    print("4. anthropic/claude-3-haiku")
    print("5. meta-llama/llama-3-70b-instruct")
    
    model_choice = input("\nEnter model name or number (default: anthropic/claude-3-haiku): ").strip()
    
    model_map = {
        "1": "openai/gpt-4",
        "2": "openai/gpt-3.5-turbo", 
        "3": "anthropic/claude-3-sonnet",
        "4": "anthropic/claude-3-haiku",
        "5": "meta-llama/llama-3-70b-instruct"
    }
    
    if model_choice in model_map:
        model = model_map[model_choice]
    elif model_choice:
        model = model_choice
    else:
        model = "anthropic/claude-3-haiku"
        
    print(f"Using model: {model}")
    
    # Run the experiment
    print("\nStarting experiment...")
    df, in_sample_results, out_of_sample_results = asyncio.run(run_full_experiment(api_key, model))
    
    # Save results
    output_file = f"phrase_analysis_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
    df.to_csv(output_file, index=False)
    print(f"\nResults saved to: {output_file}")


if __name__ == "__main__":
    main()