src/benchmarks/prompt_eval.py

import json
import os
from collections import Counter
from typing import List, Dict, Any
from src.parser.parser import Parser
from src.ontology.matcher import ConceptMatcher
from src.embeddings.engine import EmbeddingEngine
from src.enrichment.enricher import Enricher
from src.prompt_builder.builder import PromptBuilder

class PromptEvaluator:
    def __init__(self, profiles: List[str]):
        self.profiles = profiles

    def evaluate_prompts(self, count: int = 1000):
        matcher = ConceptMatcher("data/ontology")
        engine = EmbeddingEngine(index_dir="data/faiss_indices")
        engine.load_index()
        parser = Parser(matcher, engine)
        enricher = Enricher(matcher)
        
        # Sample inputs for 1000 prompts
        # We'll use the benchmark generator logic or just some variants
        from src.benchmarks.benchmark import BenchmarkSuite
        suite = BenchmarkSuite("data/ontology")
        benchmarks = suite.generate_benchmarks(count)
        
        report = {}
        
        for profile in self.profiles:
            builder = PromptBuilder(profile_name=profile)
            profile_results = {
                "duplicate_terms": 0,
                "avg_length": 0,
                "total_prompts": count,
                "concept_coverage": Counter()
            }
            
            total_len = 0
            for case in benchmarks:
                ir = parser.parse(case["input"])
                ir = enricher.enrich(ir)
                prompt_bundle = builder.build(ir)
                pos = prompt_bundle["positive"]
                
                tags = [t.strip() for t in pos.split(",")]
                if len(tags) != len(set(tags)):
                    profile_results["duplicate_terms"] += 1
                
                total_len += len(tags)
                for tag in tags:
                    profile_results["concept_coverage"][tag] += 1
                    
            profile_results["avg_length"] = total_len / count
            profile_results["top_concepts"] = dict(profile_results["concept_coverage"].most_common(10))
            del profile_results["concept_coverage"]
            
            report[profile] = profile_results
            
        return report

if __name__ == "__main__":
    evaluator = PromptEvaluator(["generic", "sdxl", "pony", "illustrious"])
    print("Evaluating 1000 prompts per profile...")
    report = evaluator.evaluate_prompts(1000)
    
    os.makedirs("reports", exist_ok=True)
    with open("reports/quality_report.json", "w", encoding="utf-8") as f:
        json.dump(report, f, indent=2)
        
    with open("reports/quality_report.md", "w", encoding="utf-8") as f:
        f.write("# Prompt Quality Evaluation\n\n")
        for profile, metrics in report.items():
            f.write(f"## Profile: {profile.upper()}\n")
            f.write(f"- **Avg Prompt Length**: {metrics['avg_length']} tags\n")
            f.write(f"- **Prompts with Duplicates**: {metrics['duplicate_terms']}\n")
            f.write(f"- **Top Concepts**: {', '.join(metrics['top_concepts'].keys())}\n\n")
            
    print("Quality report generated in reports/")