stack/training/benchmark_optimized.py

#!/usr/bin/env python3
"""
Stack 2.9 Benchmark Script
Compares optimized model against base model for speed, memory, and quality.
"""

import argparse
import os
import sys
import json
import time
import torch
import psutil
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Tuple, Optional

# Add parent to path
sys.path.insert(0, str(Path(__file__).parent.parent))


def parse_args():
    parser = argparse.ArgumentParser(description="Benchmark Stack 2.9")
    parser.add_argument(
        "--base-model",
        type=str,
        default="Qwen/Qwen2.5-Coder-32B",
        help="Base model name/path"
    )
    parser.add_argument(
        "--optimized-model",
        type=str,
        default="./output/stack-2.9-quantized",
        help="Optimized model path"
    )
    parser.add_argument(
        "--test-prompts",
        type=str,
        default=None,
        help="JSON file with test prompts"
    )
    parser.add_argument(
        "--num-runs",
        type=int,
        default=5,
        help="Number of benchmark runs per prompt"
    )
    parser.add_argument(
        "--output",
        type=str,
        default="./benchmarks/optimization_results.json",
        help="Output file for results"
    )
    parser.add_argument(
        "--test-mmlu",
        action="store_true",
        help="Run MMLU quality test"
    )
    return parser.parse_args()


def get_memory_usage() -> Dict:
    """Get current memory usage."""
    return {
        "ram_used_gb": psutil.Process().memory_info().rss / (1024**3),
        "ram_percent": psutil.Process().memory_percent(),
        "cuda_allocated_gb": torch.cuda.memory_allocated() / (1024**3) if torch.cuda.is_available() else 0,
        "cuda_reserved_gb": torch.cuda.memory_reserved() / (1024**3) if torch.cuda.is_available() else 0
    }


def get_model_size(path: str) -> float:
    """Calculate model size in GB."""
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            if os.path.exists(fp):
                total_size += os.path.getsize(fp)
    return total_size / (1024**3)


def load_model(model_path: str, quantized: bool = False):
    """Load model and tokenizer."""
    from transformers import AutoModelForCausalLM, AutoTokenizer
    
    kwargs = {
        "trust_remote_code": True
    }
    
    if quantized or "quantized" in model_path or "awq" in model_path or "bnb" in model_path:
        kwargs["torch_dtype"] = torch.float16
        kwargs["load_in_4bit"] = True
        kwargs["device_map"] = "auto"
    else:
        kwargs["torch_dtype"] = torch.bfloat16
        kwargs["device_map"] = "auto"
    
    print(f"  Loading from {model_path}...")
    model = AutoModelForCausalLM.from_pretrained(model_path, **kwargs)
    
    try:
        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
    except:
        tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-32B", trust_remote_code=True)
    
    return model, tokenizer


def benchmark_inference(
    model, 
    tokenizer, 
    prompt: str, 
    num_runs: int = 5,
    max_new_tokens: int = 100
) -> Dict:
    """Benchmark inference speed and memory."""
    if torch.cuda.is_available():
        torch.cuda.reset_peak_memory_stats()
    
    # Prepare inputs
    messages = [
        {"role": "system", "content": "You are Stack, a helpful coding assistant."},
        {"role": "user", "content": prompt}
    ]
    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    inputs = tokenizer(text, return_tensors="pt").to(model.device)
    
    # Warm up
    with torch.no_grad():
        _ = model.generate(**inputs, max_new_tokens=20, do_sample=False)
    
    if torch.cuda.is_available():
        torch.cuda.synchronize()
    
    # Benchmark
    times = []
    tokens_generated = []
    
    for i in range(num_runs):
        if torch.cuda.is_available():
            torch.cuda.synchronize()
        
        start = time.perf_counter()
        
        with torch.no_grad():
            outputs = model.generate(
                **inputs, 
                max_new_tokens=max_new_tokens, 
                do_sample=False
            )
        
        if torch.cuda.is_available():
            torch.cuda.synchronize()
        
        elapsed = time.perf_counter() - start
        times.append(elapsed)
        
        # Count generated tokens
        gen_tokens = outputs[0].shape[0] - inputs["input_ids"].shape[0]
        tokens_generated.append(gen_tokens)
    
    # Get memory stats
    if torch.cuda.is_available():
        peak_memory = torch.cuda.max_memory_allocated() / (1024**3)
    else:
        peak_memory = 0
    
    return {
        "times": times,
        "avg_time": sum(times) / len(times),
        "min_time": min(times),
        "max_time": max(times),
        "tokens_generated": tokens_generated,
        "avg_tokens": sum(tokens_generated) / len(tokens_generated),
        "tokens_per_second": sum(tokens_generated) / sum(times),
        "peak_memory_gb": peak_memory
    }


def run_mmlu_test(model, tokenizer) -> Optional[float]:
    """Run a simple MMLU subset test."""
    # MMLU is complex to set up, so we do a simple coding task quality check
    # In production, use the official MMLU evaluation
    
    print("  Running quality assessment...")
    
    coding_tasks = [
        {
            "prompt": "Write a Python function to check if a string is a palindrome.",
            "expected_keywords": ["def", "string", "reverse", "return"]
        },
        {
            "prompt": "Implement binary search in Python.",
            "expected_keywords": ["def", "left", "right", "mid", "return"]
        },
        {
            "prompt": "Create a Python class for a stack data structure.",
            "expected_keywords": ["class", "def", "__init__", "push", "pop"]
        }
    ]
    
    correct = 0
    total = len(coding_tasks)
    
    for task in coding_tasks:
        messages = [
            {"role": "user", "content": task["prompt"]}
        ]
        text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        inputs = tokenizer(text, return_tensors="pt").to(model.device)
        
        with torch.no_grad():
            outputs = model.generate(**inputs, max_new_tokens=200, do_sample=False)
        
        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
        
        # Simple keyword check
        response_lower = response.lower()
        if all(kw.lower() in response_lower for kw in task["expected_keywords"][:2]):
            correct += 1
    
    return (correct / total) * 100 if total > 0 else None


def get_default_prompts() -> List[str]:
    """Get default test prompts."""
    return [
        "Write a Python function to calculate factorial recursively.",
        "Explain what a binary tree is in simple terms.",
        "Write a SQL query to find duplicate records in a table.",
        "How do I sort a list in Python?",
        "Write a hello world program in Python."
    ]


def generate_report(results: Dict) -> str:
    """Generate a markdown report."""
    report = f"""# Stack 2.9 Optimization Benchmark Report

Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}

## Summary

| Metric | Base Model | Optimized Model | Improvement |
|--------|------------|-----------------|-------------|
| Size | {results['base']['size_gb']:.2f} GB | {results['optimized']['size_gb']:.2f} GB | {results['comparison']['size_reduction']:.1f}% smaller |
| Speed | {results['base']['avg_tokens_per_sec']:.1f} tok/s | {results['optimized']['avg_tokens_per_sec']:.1f} tok/s | {results['comparison']['speed_improvement']:.1f}x |
| Memory | {results['base']['peak_memory_gb']:.2f} GB | {results['optimized']['peak_memory_gb']:.2f} GB | {results['comparison']['memory_reduction']:.1f}% less |

## Detailed Results

### Base Model ({results['base']['path']})

- **Size**: {results['base']['size_gb']:.2f} GB
- **Avg Inference Time**: {results['base']['avg_time']:.3f}s
- **Tokens/Second**: {results['base']['avg_tokens_per_sec']:.1f}
- **Peak GPU Memory**: {results['base']['peak_memory_gb']:.2f} GB

### Optimized Model ({results['optimized']['path']})

- **Size**: {results['optimized']['size_gb']:.2f} GB
- **Avg Inference Time**: {results['optimized']['avg_time']:.3f}s
- **Tokens/Second**: {results['optimized']['avg_tokens_per_sec']:.1f}
- **Peak GPU Memory**: {results['optimized']['peak_memory_gb']:.2f} GB

### Prompt-Level Results

| Prompt | Base Time (s) | Optimized Time (s) | Speedup |
|--------|---------------|---------------------|---------|
"""
    
    for i, prompt in enumerate(results['prompts']):
        base_time = results['base']['prompt_results'][i]['avg_time']
        opt_time = results['optimized']['prompt_results'][i]['avg_time']
        speedup = base_time / opt_time if opt_time > 0 else 0
        short_prompt = prompt[:50] + "..." if len(prompt) > 50 else prompt
        report += f"| {short_prompt} | {base_time:.3f} | {opt_time:.3f} | {speedup:.2f}x |\n"
    
    report += f"""
## Quality Assessment

- **MMLU Score (Base)**: {results['base'].get('mmlu_score', 'N/A')}
- **MMLU Score (Optimized)**: {results['optimized'].get('mmlu_score', 'N/A')}

## Recommendations

"""
    
    if results['comparison']['speed_improvement'] > 1.5:
        report += "- ✅ Significant speedup achieved with quantization\n"
    if results['comparison']['memory_reduction'] > 30:
        report += "- ✅ Memory usage reduced significantly\n"
    if results['comparison']['size_reduction'] > 40:
        report += "- ✅ Model size reduced, enabling deployment on smaller hardware\n"
    
    report += """
## How to Use

```bash
# Run inference with optimized model
python convert_openai.py --model-path ./output/stack-2.9-quantized

# Or with vLLM for even better performance
vllm serve ./output/stack-2.9-quantized --dtype half
```
"""
    
    return report


def main():
    args = parse_args()
    
    print("=" * 60)
    print("Stack 2.9 Optimization Benchmark")
    print("=" * 60)
    print(f"Base model: {args.base_model}")
    print(f"Optimized model: {args.optimized_model}")
    print(f"Test runs: {args.num_runs}")
    print("=" * 60)
    
    results = {
        "timestamp": datetime.now().isoformat(),
        "base": {"path": args.base_model},
        "optimized": {"path": args.optimized_model},
        "prompts": []
    }
    
    # Get test prompts
    if args.test_prompts and os.path.exists(args.test_prompts):
        with open(args.test_prompts) as f:
            data = json.load(f)
            prompts = data.get("prompts", get_default_prompts())
    else:
        prompts = get_default_prompts()
    
    results["prompts"] = prompts
    
    # Benchmark base model
    print("\n" + "=" * 40)
    print("Benchmarking Base Model")
    print("=" * 40)
    
    try:
        if args.base_model.startswith("Qwen/"):
            from transformers import AutoModelForCausalLM, AutoTokenizer
            base_model = AutoModelForCausalLM.from_pretrained(
                args.base_model,
                torch_dtype=torch.bfloat16,
                device_map="auto",
                trust_remote_code=True
            )
            base_tokenizer = AutoTokenizer.from_pretrained(args.base_model, trust_remote_code=True)
        else:
            base_model, base_tokenizer = load_model(args.base_model, quantized=False)
        
        results["base"]["size_gb"] = get_model_size(args.base_model)
        results["base"]["prompt_results"] = []
        
        for i, prompt in enumerate(prompts):
            print(f"  [{i+1}/{len(prompts)}] {prompt[:40]}...")
            result = benchmark_inference(base_model, base_tokenizer, prompt, args.num_runs)
            results["base"]["prompt_results"].append(result)
            print(f"    Time: {result['avg_time']:.3f}s, Tokens/s: {result['tokens_per_second']:.1f}")
        
        results["base"]["avg_time"] = sum(r["avg_time"] for r in results["base"]["prompt_results"]) / len(prompts)
        results["base"]["avg_tokens_per_sec"] = sum(r["tokens_per_second"] for r in results["base"]["prompt_results"]) / len(prompts)
        results["base"]["peak_memory_gb"] = max(r["peak_memory_gb"] for r in results["base"]["prompt_results"])
        
        if args.test_mmlu:
            results["base"]["mmlu_score"] = run_mmlu_test(base_model, base_tokenizer)
        
        del base_model
        torch.cuda.empty_cache()
        
    except Exception as e:
        print(f"  Base model benchmark failed: {e}")
        results["base"]["error"] = str(e)
    
    # Benchmark optimized model
    print("\n" + "=" * 40)
    print("Benchmarking Optimized Model")
    print("=" * 40)
    
    if not os.path.exists(args.optimized_model):
        print(f"  Optimized model not found at {args.optimized_model}")
        print("  Skipping optimized benchmarks")
    else:
        try:
            opt_model, opt_tokenizer = load_model(args.optimized_model, quantized=True)
            
            results["optimized"]["size_gb"] = get_model_size(args.optimized_model)
            results["optimized"]["prompt_results"] = []
            
            for i, prompt in enumerate(prompts):
                print(f"  [{i+1}/{len(prompts)}] {prompt[:40]}...")
                result = benchmark_inference(opt_model, opt_tokenizer, prompt, args.num_runs)
                results["optimized"]["prompt_results"].append(result)
                print(f"    Time: {result['avg_time']:.3f}s, Tokens/s: {result['tokens_per_second']:.1f}")
            
            results["optimized"]["avg_time"] = sum(r["avg_time"] for r in results["optimized"]["prompt_results"]) / len(prompts)
            results["optimized"]["avg_tokens_per_sec"] = sum(r["tokens_per_second"] for r in results["optimized"]["prompt_results"]) / len(prompts)
            results["optimized"]["peak_memory_gb"] = max(r["peak_memory_gb"] for r in results["optimized"]["prompt_results"])
            
            if args.test_mmlu:
                results["optimized"]["mmlu_score"] = run_mmlu_test(opt_model, opt_tokenizer)
            
            del opt_model
            torch.cuda.empty_cache()
            
        except Exception as e:
            print(f"  Optimized model benchmark failed: {e}")
            results["optimized"]["error"] = str(e)
    
    # Generate comparison
    if "size_gb" in results["base"] and "size_gb" in results["optimized"]:
        results["comparison"] = {
            "size_reduction": (1 - results["optimized"]["size_gb"] / results["base"]["size_gb"]) * 100,
            "speed_improvement": results["optimized"]["avg_tokens_per_sec"] / results["base"]["avg_tokens_per_sec"] if results["base"]["avg_tokens_per_sec"] > 0 else 0,
            "memory_reduction": (1 - results["optimized"]["peak_memory_gb"] / results["base"]["peak_memory_gb"]) * 100 if results["base"]["peak_memory_gb"] > 0 else 0
        }
    
    # Save results
    os.makedirs(os.path.dirname(args.output), exist_ok=True)
    with open(args.output, "w") as f:
        json.dump(results, f, indent=2)
    
    # Generate and save report
    if "comparison" in results:
        report = generate_report(results)
        report_path = args.output.replace(".json", "_report.md")
        with open(report_path, "w") as f:
            f.write(report)
        print(f"\n📊 Report saved to: {report_path}")
    
    print(f"\n📊 Results saved to: {args.output}")
    
    # Print summary
    if "comparison" in results:
        print("\n" + "=" * 60)
        print("SUMMARY")
        print("=" * 60)
        print(f"Size reduction: {results['comparison']['size_reduction']:.1f}%")
        print(f"Speed improvement: {results['comparison']['speed_improvement']:.2f}x")
        print(f"Memory reduction: {results['comparison']['memory_reduction']:.1f}%")
    
    return 0


if __name__ == "__main__":
    sys.exit(main())