analyze_comprehensive_final.py

#!/usr/bin/env python3
"""
Final Comprehensive Analysis Script
Analyzes the comprehensive test results: 256 models × 20 tests
Generates all figures and tables for the paper
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
from datetime import datetime
import json

# Set style for publication-quality figures
plt.style.use('seaborn-v0_8-whitegrid')
sns.set_palette("husl")

def load_results():
    """Load the comprehensive test results."""
    excel_file = 'comprehensive_20_tests_results_20251014_153008.xlsx'
    json_file = 'comprehensive_20_tests_results_20251014_153008.json'

    print("="*80)
    print("LOADING COMPREHENSIVE TEST RESULTS")
    print("="*80)
    print(f"\nLoading from: {excel_file}")

    # Load Excel file
    xls = pd.ExcelFile(excel_file)

    # Load all sheets
    all_results = pd.read_excel(xls, 'All Results')
    model_rankings = pd.read_excel(xls, 'Model Rankings', index_col=0)
    test_difficulty = pd.read_excel(xls, 'Test Difficulty')
    category_performance = pd.read_excel(xls, 'Category Performance', index_col=0)

    print(f"  Total results: {len(all_results)}")
    print(f"  Models tested: {all_results['model'].nunique()}")
    print(f"  Tests conducted: {all_results['test_id'].nunique()}")

    # Load JSON for additional metadata
    with open(json_file, 'r') as f:
        json_data = json.load(f)

    return all_results, model_rankings, test_difficulty, category_performance, json_data

def print_summary_statistics(all_results, json_data):
    """Print comprehensive summary statistics."""
    print("\n" + "="*80)
    print("SUMMARY STATISTICS")
    print("="*80)

    metadata = json_data['metadata']
    summary = json_data['summary']

    print(f"\nDataset Overview:")
    print(f"  Total Models: {metadata['total_models']}")
    print(f"  Total Tests: {metadata['total_tests']}")
    print(f"  Total Evaluations: {metadata['total_results']}")
    print(f"  Timestamp: {metadata['timestamp']}")

    print(f"\nOverall Performance:")
    print(f"  Overall Pass Rate: {summary['overall_pass_rate']:.1f}%")
    print(f"  Best Model: {summary['best_model']} ({summary['best_model_score']:.1f}%)")
    print(f"  Hardest Test: Test {summary['hardest_test']} ({summary['hardest_test_pass_rate']:.1f}% pass rate)")

    # API success rate
    success_rate = (all_results['status'] == 'success').mean() * 100
    print(f"  API Success Rate: {success_rate:.1f}%")

    # Response time statistics
    avg_response_time = all_results[all_results['response_time'] > 0]['response_time'].mean()
    print(f"  Average Response Time: {avg_response_time:.2f}s")

def print_top_models(model_rankings):
    """Print top performing models."""
    print("\n" + "="*80)
    print("TOP 20 PERFORMING MODELS")
    print("="*80)

    print(f"\n{'Rank':<6} {'Pass Rate':<12} {'Model'}")
    print("-" * 80)

    for idx, (model, row) in enumerate(model_rankings.head(20).iterrows(), 1):
        pass_rate = row['Pass Rate (%)']
        print(f"{idx:<6} {pass_rate:>6.1f}%      {model}")

def print_test_difficulty(test_difficulty):
    """Print test difficulty analysis."""
    print("\n" + "="*80)
    print("TEST DIFFICULTY ANALYSIS (HARDEST TO EASIEST)")
    print("="*80)

    print(f"\n{'ID':<4} {'Pass Rate':<12} {'Category':<25} {'Test Name'}")
    print("-" * 95)

    # Sort by pass rate (ascending = hardest first)
    sorted_tests = test_difficulty.sort_values('Pass Rate (%)')

    for _, row in sorted_tests.iterrows():
        test_id = int(row['Test ID'])
        pass_rate = row['Pass Rate (%)']
        category = row['category'][:23] if pd.notna(row['category']) else ''
        name = row['name'][:45]
        print(f"{test_id:<4} {pass_rate:>6.1f}%      {category:<25} {name}")

def print_category_analysis(category_performance):
    """Print category performance analysis."""
    print("\n" + "="*80)
    print("PERFORMANCE BY CATEGORY")
    print("="*80)

    # Sort by pass rate
    sorted_cats = category_performance.sort_values('Pass Rate (%)')

    print(f"\n{'Category':<30} {'Pass Rate'}")
    print("-" * 45)

    for category, row in sorted_cats.iterrows():
        pass_rate = row['Pass Rate (%)']
        print(f"{category:<30} {pass_rate:>6.1f}%")

def analyze_by_provider(all_results):
    """Analyze performance by model provider."""
    print("\n" + "="*80)
    print("PROVIDER ANALYSIS")
    print("="*80)

    # Extract provider from model name
    all_results['provider'] = all_results['model'].apply(
        lambda x: x.split('/')[0] if '/' in x else 'other'
    )

    # Calculate provider statistics
    provider_stats = all_results.groupby('provider').agg({
        'passed': 'mean',
        'model': 'nunique',
        'test_id': 'count'
    }).round(3)

    provider_stats.columns = ['pass_rate', 'num_models', 'total_tests']
    provider_stats['pass_rate'] = provider_stats['pass_rate'] * 100
    provider_stats = provider_stats.sort_values('pass_rate', ascending=False)

    # Filter to providers with at least 3 models
    provider_stats_filtered = provider_stats[provider_stats['num_models'] >= 3]

    print(f"\n{'Provider':<20} {'Pass Rate':<12} {'Models':<10} {'Tests'}")
    print("-" * 55)

    for provider, row in provider_stats_filtered.head(15).iterrows():
        print(f"{provider:<20} {row['pass_rate']:>6.1f}%      {int(row['num_models']):<10} {int(row['total_tests'])}")

    return provider_stats_filtered

def create_heatmap(all_results, output_file='fig1_heatmap.pdf'):
    """Create heatmap of model performance on all tests."""
    print("\n" + "="*80)
    print("CREATING FIGURE 1: PERFORMANCE HEATMAP (Top 50 Models)")
    print("="*80)

    # Create pivot table
    pivot = all_results.pivot_table(
        index='model',
        columns='test_id',
        values='passed',
        aggfunc='first'
    )

    # Select top 50 models by overall performance
    model_scores = pivot.mean(axis=1).sort_values(ascending=False)
    top_50_models = model_scores.head(50).index
    pivot_top50 = pivot.loc[top_50_models]

    # Create figure
    fig, ax = plt.subplots(figsize=(14, 12))

    # Create heatmap
    sns.heatmap(
        pivot_top50,
        cmap=['#d73027', '#91cf60'],  # Red for fail, green for pass
        cbar_kws={'label': 'Pass (1) / Fail (0)', 'ticks': [0, 1]},
        xticklabels=True,
        yticklabels=True,
        vmin=0,
        vmax=1,
        linewidths=0.3,
        linecolor='gray',
        ax=ax
    )

    plt.title('Model Performance on 20 Diagnostic Tests (Top 50 Models)',
             fontsize=16, fontweight='bold', pad=20)
    plt.xlabel('Test ID', fontsize=12)
    plt.ylabel('Model', fontsize=12)
    plt.yticks(fontsize=7)
    plt.xticks(fontsize=10)
    plt.tight_layout()

    plt.savefig(output_file, dpi=300, bbox_inches='tight')
    print(f"✅ Saved: {output_file}")
    plt.close()

def create_provider_chart(provider_stats, output_file='fig2_provider.pdf'):
    """Create bar chart of provider performance."""
    print("\n" + "="*80)
    print("CREATING FIGURE 2: PROVIDER COMPARISON")
    print("="*80)

    # Select top 12 providers
    top_providers = provider_stats.head(12)

    # Create figure
    fig, ax = plt.subplots(figsize=(12, 7))

    # Create bar plot
    x_pos = np.arange(len(top_providers))
    colors = sns.color_palette('husl', len(top_providers))
    bars = ax.bar(x_pos, top_providers['pass_rate'], color=colors, edgecolor='black', linewidth=0.5)

    # Customize plot
    ax.set_xlabel('Model Provider', fontsize=13, fontweight='bold')
    ax.set_ylabel('Average Pass Rate (%)', fontsize=13, fontweight='bold')
    ax.set_title('Performance by Model Provider (≥3 models)',
                fontsize=16, fontweight='bold', pad=20)
    ax.set_xticks(x_pos)
    ax.set_xticklabels(top_providers.index, rotation=45, ha='right', fontsize=11)
    ax.set_ylim([0, max(top_providers['pass_rate']) * 1.15])
    ax.grid(axis='y', alpha=0.3)

    # Add value labels on bars
    for bar in bars:
        height = bar.get_height()
        ax.text(bar.get_x() + bar.get_width()/2., height + 1,
               f'{height:.1f}%', ha='center', va='bottom', fontsize=10, fontweight='bold')

    # Add average line
    avg_rate = top_providers['pass_rate'].mean()
    ax.axhline(y=avg_rate, color='red', linestyle='--', alpha=0.7, linewidth=2,
              label=f'Average: {avg_rate:.1f}%')
    ax.legend(fontsize=11)

    plt.tight_layout()
    plt.savefig(output_file, dpi=300, bbox_inches='tight')
    print(f"✅ Saved: {output_file}")
    plt.close()

def create_difficulty_chart(test_difficulty, output_file='fig3_difficulty.pdf'):
    """Create horizontal bar chart of test difficulty."""
    print("\n" + "="*80)
    print("CREATING FIGURE 3: TEST DIFFICULTY RANKING")
    print("="*80)

    # Sort by difficulty (ascending pass rate = harder)
    sorted_tests = test_difficulty.sort_values('Pass Rate (%)')

    # Create shortened labels
    labels = []
    for _, row in sorted_tests.iterrows():
        test_id = int(row['Test ID'])
        name = row['name']
        # Shorten long names
        if len(name) > 35:
            name = name[:32] + '...'
        labels.append(f"T{test_id}: {name}")

    pass_rates = sorted_tests['Pass Rate (%)'].values

    # Create figure
    fig, ax = plt.subplots(figsize=(12, 10))

    # Color based on difficulty
    colors = []
    for rate in pass_rates:
        if rate < 10:
            colors.append('#8B0000')  # Dark red - extremely hard
        elif rate < 20:
            colors.append('#d73027')  # Red - very hard
        elif rate < 40:
            colors.append('#fdae61')  # Orange - hard
        elif rate < 60:
            colors.append('#fee08b')  # Yellow - medium
        elif rate < 80:
            colors.append('#a6d96a')  # Light green - easy
        else:
            colors.append('#1a9850')  # Dark green - very easy

    # Create horizontal bar plot
    y_pos = np.arange(len(labels))
    bars = ax.barh(y_pos, pass_rates, color=colors, edgecolor='black', linewidth=0.5)

    # Customize plot
    ax.set_xlabel('Pass Rate (%)', fontsize=13, fontweight='bold')
    ax.set_ylabel('Test', fontsize=13, fontweight='bold')
    ax.set_title('Test Difficulty Ranking (Hardest to Easiest)',
                fontsize=16, fontweight='bold', pad=20)
    ax.set_yticks(y_pos)
    ax.set_yticklabels(labels, fontsize=9)
    ax.set_xlim([0, 105])
    ax.grid(axis='x', alpha=0.3)

    # Add value labels
    for bar, rate in zip(bars, pass_rates):
        width = bar.get_width()
        ax.text(width + 1.5, bar.get_y() + bar.get_height()/2.,
               f'{rate:.1f}%', ha='left', va='center', fontsize=9, fontweight='bold')

    # Add vertical reference lines
    ax.axvline(x=50, color='black', linestyle='--', alpha=0.3, linewidth=1)
    ax.axvline(x=25, color='red', linestyle=':', alpha=0.3, linewidth=1)
    ax.axvline(x=75, color='green', linestyle=':', alpha=0.3, linewidth=1)

    # Add legend for difficulty colors
    from matplotlib.patches import Patch
    legend_elements = [
        Patch(facecolor='#8B0000', label='Extremely Hard (<10%)'),
        Patch(facecolor='#d73027', label='Very Hard (10-20%)'),
        Patch(facecolor='#fdae61', label='Hard (20-40%)'),
        Patch(facecolor='#fee08b', label='Medium (40-60%)'),
        Patch(facecolor='#a6d96a', label='Easy (60-80%)'),
        Patch(facecolor='#1a9850', label='Very Easy (>80%)')
    ]
    ax.legend(handles=legend_elements, loc='lower right', fontsize=9)

    plt.tight_layout()
    plt.savefig(output_file, dpi=300, bbox_inches='tight')
    print(f"✅ Saved: {output_file}")
    plt.close()

def create_category_chart(category_performance, output_file='fig4_category.pdf'):
    """Create bar chart of category performance."""
    print("\n" + "="*80)
    print("CREATING FIGURE 4: CATEGORY PERFORMANCE")
    print("="*80)

    # Sort by pass rate
    sorted_cats = category_performance.sort_values('Pass Rate (%)')

    # Create figure
    fig, ax = plt.subplots(figsize=(10, 6))

    # Create bar plot
    x_pos = np.arange(len(sorted_cats))
    colors = plt.cm.RdYlGn(sorted_cats['Pass Rate (%)'] / 100)
    bars = ax.bar(x_pos, sorted_cats['Pass Rate (%)'], color=colors, edgecolor='black', linewidth=0.8)

    # Customize plot
    ax.set_xlabel('Category', fontsize=13, fontweight='bold')
    ax.set_ylabel('Pass Rate (%)', fontsize=13, fontweight='bold')
    ax.set_title('Performance by Test Category',
                fontsize=16, fontweight='bold', pad=20)
    ax.set_xticks(x_pos)
    ax.set_xticklabels(sorted_cats.index, rotation=45, ha='right', fontsize=11)
    ax.set_ylim([0, 100])
    ax.grid(axis='y', alpha=0.3)

    # Add value labels
    for bar in bars:
        height = bar.get_height()
        ax.text(bar.get_x() + bar.get_width()/2., height + 2,
               f'{height:.1f}%', ha='center', va='bottom', fontsize=11, fontweight='bold')

    plt.tight_layout()
    plt.savefig(output_file, dpi=300, bbox_inches='tight')
    print(f"✅ Saved: {output_file}")
    plt.close()

def generate_latex_tables(model_rankings, test_difficulty, category_performance, provider_stats):
    """Generate LaTeX tables for the paper."""
    print("\n" + "="*80)
    print("GENERATING LATEX TABLES")
    print("="*80)

    output_file = 'paper_tables.tex'

    with open(output_file, 'w') as f:
        # Table 1: Top Models
        f.write("% Table 1: Top 15 Performing Models\n")
        f.write("\\begin{table}[htbp]\n")
        f.write("\\centering\n")
        f.write("\\caption{Top 15 Performing Models on 20 Diagnostic Tests}\n")
        f.write("\\label{tab:top_models}\n")
        f.write("\\begin{tabular}{rlr}\n")
        f.write("\\toprule\n")
        f.write("Rank & Model & Pass Rate \\\\\n")
        f.write("\\midrule\n")

        for idx, (model, row) in enumerate(model_rankings.head(15).iterrows(), 1):
            model_escaped = model.replace('_', '\\_').replace('&', '\\&')
            pass_rate = row['Pass Rate (%)']
            f.write(f"{idx} & \\texttt{{{model_escaped}}} & {pass_rate:.1f}\\% \\\\\n")

        f.write("\\bottomrule\n")
        f.write("\\end{tabular}\n")
        f.write("\\end{table}\n\n")

        # Table 2: Test Difficulty
        f.write("% Table 2: Test Difficulty (Hardest 10)\n")
        f.write("\\begin{table}[htbp]\n")
        f.write("\\centering\n")
        f.write("\\caption{Test Difficulty Analysis (10 Hardest Tests)}\n")
        f.write("\\label{tab:test_difficulty}\n")
        f.write("\\begin{tabular}{clrl}\n")
        f.write("\\toprule\n")
        f.write("ID & Test Name & Pass Rate & Category \\\\\n")
        f.write("\\midrule\n")

        sorted_tests = test_difficulty.sort_values('Pass Rate (%)')
        for _, row in sorted_tests.head(10).iterrows():
            test_id = int(row['Test ID'])
            name = row['name'][:40]
            name_escaped = name.replace('_', '\\_').replace('&', '\\&')
            pass_rate = row['Pass Rate (%)']
            category = row['category'][:20] if pd.notna(row['category']) else ''
            category_escaped = category.replace('_', '\\_').replace('&', '\\&')
            f.write(f"{test_id} & {name_escaped} & {pass_rate:.1f}\\% & {category_escaped} \\\\\n")

        f.write("\\bottomrule\n")
        f.write("\\end{tabular}\n")
        f.write("\\end{table}\n\n")

        # Table 3: Category Performance
        f.write("% Table 3: Category Performance\n")
        f.write("\\begin{table}[htbp]\n")
        f.write("\\centering\n")
        f.write("\\caption{Performance by Test Category}\n")
        f.write("\\label{tab:category_performance}\n")
        f.write("\\begin{tabular}{lr}\n")
        f.write("\\toprule\n")
        f.write("Category & Pass Rate \\\\\n")
        f.write("\\midrule\n")

        sorted_cats = category_performance.sort_values('Pass Rate (%)')
        for category, row in sorted_cats.iterrows():
            category_escaped = category.replace('_', '\\_').replace('&', '\\&')
            pass_rate = row['Pass Rate (%)']
            f.write(f"{category_escaped} & {pass_rate:.1f}\\% \\\\\n")

        f.write("\\bottomrule\n")
        f.write("\\end{tabular}\n")
        f.write("\\end{table}\n\n")

        # Table 4: Provider Comparison
        f.write("% Table 4: Provider Comparison (Top 10)\n")
        f.write("\\begin{table}[htbp]\n")
        f.write("\\centering\n")
        f.write("\\caption{Performance by Model Provider}\n")
        f.write("\\label{tab:provider_comparison}\n")
        f.write("\\begin{tabular}{lrrr}\n")
        f.write("\\toprule\n")
        f.write("Provider & Models & Pass Rate & Tests \\\\\n")
        f.write("\\midrule\n")

        for provider, row in provider_stats.head(10).iterrows():
            provider_escaped = provider.replace('_', '\\_').replace('&', '\\&')
            num_models = int(row['num_models'])
            pass_rate = row['pass_rate']
            total_tests = int(row['total_tests'])
            f.write(f"{provider_escaped} & {num_models} & {pass_rate:.1f}\\% & {total_tests} \\\\\n")

        f.write("\\bottomrule\n")
        f.write("\\end{tabular}\n")
        f.write("\\end{table}\n")

    print(f"✅ Saved: {output_file}")

def generate_summary_report(all_results, model_rankings, test_difficulty, category_performance, provider_stats, json_data):
    """Generate comprehensive summary report."""
    print("\n" + "="*80)
    print("GENERATING FINAL SUMMARY REPORT")
    print("="*80)

    output_file = 'FINAL_RESULTS_SUMMARY.md'

    with open(output_file, 'w') as f:
        f.write("# Comprehensive LLM Instruction Following Evaluation\n")
        f.write(f"## Final Results Summary\n\n")
        f.write(f"**Analysis Date:** {datetime.now().strftime('%Y-%m-%d %H:%M')}\n\n")
        f.write(f"**Data Source:** {json_data['metadata']['timestamp']}\n\n")

        f.write("---\n\n")
        f.write("## Executive Summary\n\n")

        metadata = json_data['metadata']
        summary = json_data['summary']

        f.write(f"This evaluation tested **{metadata['total_models']} large language models** ")
        f.write(f"on **{metadata['total_tests']} diagnostic prompts**, ")
        f.write(f"resulting in **{metadata['total_results']} individual test evaluations**.\n\n")

        f.write(f"### Key Findings\n\n")
        f.write(f"- **Overall Pass Rate:** {summary['overall_pass_rate']:.1f}%\n")
        f.write(f"- **Best Performing Model:** {summary['best_model']} ({summary['best_model_score']:.1f}%)\n")
        f.write(f"- **Hardest Test:** Test {summary['hardest_test']} ({summary['hardest_test_pass_rate']:.1f}% pass rate)\n")

        # API success rate
        success_rate = (all_results['status'] == 'success').mean() * 100
        f.write(f"- **API Success Rate:** {success_rate:.1f}%\n\n")

        f.write("---\n\n")
        f.write("## Top 20 Performing Models\n\n")
        f.write("| Rank | Model | Pass Rate |\n")
        f.write("|------|-------|----------:|\n")

        for idx, (model, row) in enumerate(model_rankings.head(20).iterrows(), 1):
            pass_rate = row['Pass Rate (%)']
            f.write(f"| {idx} | {model} | {pass_rate:.1f}% |\n")

        f.write("\n---\n\n")
        f.write("## Test Difficulty Analysis\n\n")
        f.write("### Hardest Tests (Lowest Pass Rates)\n\n")
        f.write("| ID | Test Name | Category | Pass Rate |\n")
        f.write("|----|-----------|----------|----------:|\n")

        sorted_tests = test_difficulty.sort_values('Pass Rate (%)')
        for _, row in sorted_tests.head(10).iterrows():
            test_id = int(row['Test ID'])
            name = row['name']
            category = row['category'] if pd.notna(row['category']) else 'N/A'
            pass_rate = row['Pass Rate (%)']
            f.write(f"| {test_id} | {name} | {category} | {pass_rate:.1f}% |\n")

        f.write("\n### Easiest Tests (Highest Pass Rates)\n\n")
        f.write("| ID | Test Name | Category | Pass Rate |\n")
        f.write("|----|-----------|----------|----------:|\n")

        for _, row in sorted_tests.tail(10).iterrows():
            test_id = int(row['Test ID'])
            name = row['name']
            category = row['category'] if pd.notna(row['category']) else 'N/A'
            pass_rate = row['Pass Rate (%)']
            f.write(f"| {test_id} | {name} | {category} | {pass_rate:.1f}% |\n")

        f.write("\n---\n\n")
        f.write("## Performance by Category\n\n")
        f.write("| Category | Pass Rate |\n")
        f.write("|----------|----------:|\n")

        sorted_cats = category_performance.sort_values('Pass Rate (%)')
        for category, row in sorted_cats.iterrows():
            pass_rate = row['Pass Rate (%)']
            f.write(f"| {category} | {pass_rate:.1f}% |\n")

        f.write("\n**Key Insight:** String manipulation tests are by far the hardest category, ")
        f.write("while constraint compliance tests are the easiest.\n\n")

        f.write("---\n\n")
        f.write("## Performance by Provider\n\n")
        f.write("Top providers (with ≥3 models):\n\n")
        f.write("| Provider | Models | Pass Rate | Total Tests |\n")
        f.write("|----------|--------|-----------|------------:|\n")

        for provider, row in provider_stats.head(15).iterrows():
            num_models = int(row['num_models'])
            pass_rate = row['pass_rate']
            total_tests = int(row['total_tests'])
            f.write(f"| {provider} | {num_models} | {pass_rate:.1f}% | {total_tests} |\n")

        f.write("\n---\n\n")
        f.write("## Statistical Insights\n\n")

        # Calculate additional statistics
        model_pass_rates = all_results.groupby('model')['passed'].mean() * 100
        test_pass_rates = all_results.groupby('test_id')['passed'].mean() * 100

        f.write(f"### Model Performance Distribution\n\n")
        f.write(f"- **Mean Pass Rate:** {model_pass_rates.mean():.1f}%\n")
        f.write(f"- **Median Pass Rate:** {model_pass_rates.median():.1f}%\n")
        f.write(f"- **Standard Deviation:** {model_pass_rates.std():.1f}%\n")
        f.write(f"- **Models with 100% Pass Rate:** {(model_pass_rates == 100).sum()}\n")
        f.write(f"- **Models with 0% Pass Rate:** {(model_pass_rates == 0).sum()}\n\n")

        f.write(f"### Test Difficulty Distribution\n\n")
        f.write(f"- **Mean Test Pass Rate:** {test_pass_rates.mean():.1f}%\n")
        f.write(f"- **Median Test Pass Rate:** {test_pass_rates.median():.1f}%\n")
        f.write(f"- **Standard Deviation:** {test_pass_rates.std():.1f}%\n")
        f.write(f"- **Tests with >80% Pass Rate:** {(test_pass_rates > 80).sum()}\n")
        f.write(f"- **Tests with <20% Pass Rate:** {(test_pass_rates < 20).sum()}\n\n")

        f.write("---\n\n")
        f.write("## Files Generated\n\n")
        f.write("- `fig1_heatmap.pdf` - Performance heatmap (top 50 models)\n")
        f.write("- `fig2_provider.pdf` - Provider comparison chart\n")
        f.write("- `fig3_difficulty.pdf` - Test difficulty ranking\n")
        f.write("- `fig4_category.pdf` - Category performance chart\n")
        f.write("- `paper_tables.tex` - LaTeX tables for paper\n")
        f.write("- `FINAL_RESULTS_SUMMARY.md` - This summary document\n\n")

        f.write("---\n\n")
        f.write("## Conclusions\n\n")
        f.write("1. **Model Performance Varies Widely:** Pass rates range from 0% to 100%, ")
        f.write("indicating significant differences in instruction-following capabilities.\n\n")

        f.write("2. **String Manipulation is Hardest:** Tests requiring precise string manipulation ")
        f.write("have the lowest pass rates, suggesting this is a key challenge for current LLMs.\n\n")

        f.write("3. **Provider Differences:** Significant variation exists between model providers, ")
        f.write("with top providers achieving much higher pass rates.\n\n")

        f.write("4. **Few Perfect Models:** Only a small number of models achieve 100% pass rate, ")
        f.write("indicating that even top models struggle with some tests.\n\n")

        f.write("5. **API Reliability:** High API success rate indicates robust testing methodology.\n\n")

    print(f"✅ Saved: {output_file}")

def main():
    """Main analysis function."""
    print("\n" + "="*80)
    print("COMPREHENSIVE FINAL ANALYSIS")
    print(f"Analysis Date: {datetime.now().strftime('%Y-%m-%d %H:%M')}")
    print("="*80 + "\n")

    # Load results
    all_results, model_rankings, test_difficulty, category_performance, json_data = load_results()

    # Print summary statistics
    print_summary_statistics(all_results, json_data)
    print_top_models(model_rankings)
    print_test_difficulty(test_difficulty)
    print_category_analysis(category_performance)

    # Analyze by provider
    provider_stats = analyze_by_provider(all_results)

    # Create all visualizations
    create_heatmap(all_results)
    create_provider_chart(provider_stats)
    create_difficulty_chart(test_difficulty)
    create_category_chart(category_performance)

    # Generate LaTeX tables
    generate_latex_tables(model_rankings, test_difficulty, category_performance, provider_stats)

    # Generate summary report
    generate_summary_report(all_results, model_rankings, test_difficulty, category_performance, provider_stats, json_data)

    print("\n" + "="*80)
    print("✅ COMPREHENSIVE ANALYSIS COMPLETE")
    print("="*80)
    print("\nAll figures, tables, and reports have been generated successfully!")

if __name__ == "__main__":
    main()