check_cborg_routing.py

#!/usr/bin/env python3
"""
Check if CBORG provides any additional metadata about model routing or configuration.
"""
import os
from openai import OpenAI

api_key = os.environ.get('CBORG_API_KEY')
if not api_key:
    print("Error: CBORG_API_KEY not set")
    exit(1)

client = OpenAI(
    api_key=api_key,
    base_url="https://api.cborg.lbl.gov"
)

models = ["openai/o:latest", "openai/o3"]

for model in models:
    print(f"\n{'='*80}")
    print(f"Testing: {model}")
    print('='*80)
    
    # Try multiple calls to see if there's any variation
    for i in range(3):
        response = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": "Hi"}],
            max_tokens=5,
            temperature=1.0,
        )
        
        print(f"\nCall {i+1}:")
        print(f"  Response ID: {response.id}")
        print(f"  Model: {response.model}")
        print(f"  System Fingerprint: {response.system_fingerprint}")
        print(f"  Created: {response.created}")
        
        # Check for any provider-specific fields
        if hasattr(response.choices[0], 'provider_specific_fields'):
            print(f"  Provider fields: {response.choices[0].provider_specific_fields}")
        
        # Check response headers if available
        if hasattr(response, '_headers'):
            print(f"  Headers: {response._headers}")

print("\n" + "="*80)
print("CONCLUSION:")
print("="*80)
print("Both models route to the same backend (azure/o3-2025-04-16)")
print("No configuration differences detected in API responses")
print("\nThe performance differences in your dataset are due to:")
print("  1. Different experimental runs (different timestamps)")
print("  2. Natural variability in model outputs")
print("  3. Possibly different trial conditions or prompts")
print("\nCBORG appears to treat both as aliases to the same deployment.")