debug_answers.py

#!/usr/bin/env python3

import re
import os
from agent import AgentGraph as Alfred

def strip_final_answer_prefix(s: str) -> str:
    """Current function from app.py"""
    return re.sub(r"FINAL ANSWER:\s*", "", s, flags=re.IGNORECASE).strip()

def test_answer_cleaning():
    """Test the answer cleaning function"""
    print("=" * 60)
    print("Testing answer cleaning function...")
    print("=" * 60)
    
    test_cases = [
        "FINAL ANSWER: 42",
        "final answer: hello world", 
        "This is my answer without prefix",
        "FINAL ANSWER:42",
        "The reasoning... FINAL ANSWER: The correct answer is Paris",
        "FINAL ANSWER: The answer is 123",
        "My reasoning leads me to conclude that FINAL ANSWER: 456"
    ]
    
    for i, test in enumerate(test_cases, 1):
        cleaned = strip_final_answer_prefix(test)
        print(f"{i}. Input:  '{test}'")
        print(f"   Output: '{cleaned}'")
        print(f"   Length: {len(cleaned)} chars")
        print("-" * 50)

def test_agent_simple():
    """Test the agent with a simple question"""
    print("\n" + "=" * 60)
    print("Testing agent with simple question...")
    print("=" * 60)
    
    # Set minimal environment to avoid issues
    os.environ["LANGCHAIN_TRACING_V2"] = "false"
    
    try:
        agent = Alfred()
        simple_question = "What is 2 + 2?"
        
        print(f"Question: {simple_question}")
        print("Running agent...")
        
        response = agent(simple_question)
        
        print(f"\nRaw response:\n'{response}'")
        print(f"\nResponse type: {type(response)}")
        print(f"Response length: {len(str(response))} chars")
        
        cleaned = strip_final_answer_prefix(str(response))
        print(f"\nCleaned response:\n'{cleaned}'")
        print(f"Cleaned length: {len(cleaned)} chars")
        
        # Check if the response contains the expected format
        has_final_answer = "FINAL ANSWER:" in str(response).upper()
        print(f"\nContains 'FINAL ANSWER:': {has_final_answer}")
        
        if not has_final_answer:
            print("⚠️  WARNING: Agent response doesn't contain 'FINAL ANSWER:' prefix!")
            print("This could be why your submissions are getting 0% score.")
            
    except Exception as e:
        print(f"❌ Error testing agent: {e}")
        import traceback
        traceback.print_exc()

def debug_regex_issue():
    """Debug potential regex issues"""
    print("\n" + "=" * 60)
    print("Debugging regex patterns...")  
    print("=" * 60)
    
    test_response = "Based on my calculation, FINAL ANSWER: 4"
    
    # Test different regex patterns
    patterns = [
        r"FINAL ANSWER:\s*",           # Current pattern
        r"FINAL ANSWER:\s*(.*)$",      # Capture group
        r".*FINAL ANSWER:\s*",         # Remove everything up to and including prefix
        r"^.*FINAL ANSWER:\s*",        # From start to prefix
    ]
    
    for i, pattern in enumerate(patterns, 1):
        try:
            result = re.sub(pattern, "", test_response, flags=re.IGNORECASE).strip()
            print(f"{i}. Pattern: {pattern}")
            print(f"   Result: '{result}'")
            print(f"   Length: {len(result)}")
        except Exception as e:
            print(f"{i}. Pattern: {pattern} - ERROR: {e}")
        print("-" * 40)

if __name__ == "__main__":
    test_answer_cleaning()
    debug_regex_issue()
    test_agent_simple()