eval/feedback_eval.py

"""
eval/feedback_eval.py — Quantitative before/after feedback evaluation

Measures 4 metrics before and after simulated operator edits:
  1. uncited_ratio — % of sentences without citations
  2. pattern_coverage — % of learned patterns reflected in new drafts
  3. section_completeness — % of expected sections present
  4. red_flag_detection — count of red flags identified

Produces a comparison table for eval_results.md.
"""

import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))

import numpy as np
from sentence_transformers import SentenceTransformer
from loguru import logger

from config import EMBEDDING_MODEL


def compute_pattern_coverage(
    draft_text: str,
    active_patterns: list[str],
    threshold: float = 0.7,
) -> float:
    """
    Check what fraction of active patterns are reflected in the draft.

    Uses embedding similarity: if any sentence in the draft has
    similarity > threshold with the pattern, it counts as covered.
    """
    if not active_patterns:
        return 0.0

    model = SentenceTransformer(EMBEDDING_MODEL)

    # Split draft into sentences
    import re
    sentences = [s.strip() for s in re.split(r"[.!?\n]+", draft_text) if len(s.strip()) > 15]

    if not sentences:
        return 0.0

    pattern_embs = model.encode(active_patterns, normalize_embeddings=True)
    sentence_embs = model.encode(sentences, normalize_embeddings=True)

    covered = 0
    for p_emb in pattern_embs:
        sims = np.dot(sentence_embs, p_emb)
        if np.max(sims) >= threshold:
            covered += 1

    coverage = covered / len(active_patterns)
    return round(coverage, 3)


def count_red_flags(draft_text: str) -> int:
    """Count red flags mentioned in the draft."""
    lower = draft_text.lower()
    count = 0
    markers = ["red flag", "⚠", "warning", "risk", "concern", "issue identified"]
    for marker in markers:
        count += lower.count(marker)
    return count


def build_eval_table(metrics_list: list[dict]) -> str:
    """Build a markdown table from a list of metric snapshots."""
    headers = ["Metric"] + [m.get("label", f"Step {i}") for i, m in enumerate(metrics_list)]
    rows = [
        ["uncited_ratio"] + [f"{m['uncited_ratio']:.2%}" for m in metrics_list],
        ["pattern_coverage"] + [f"{m['pattern_coverage']:.0%}" for m in metrics_list],
        ["section_completeness"] + [f"{m['section_completeness']:.0%}" for m in metrics_list],
        ["red_flag_detection"] + [str(m["red_flag_count"]) for m in metrics_list],
        ["active_patterns"] + [str(m["active_patterns"]) for m in metrics_list],
    ]

    # Format as markdown table
    header_line = "| " + " | ".join(headers) + " |"
    sep_line = "| " + " | ".join(["---"] * len(headers)) + " |"
    data_lines = ["| " + " | ".join(row) + " |" for row in rows]

    return "\n".join([header_line, sep_line] + data_lines)


if __name__ == "__main__":
    # Example standalone usage
    example_metrics = [
        {"label": "Before", "uncited_ratio": 0.25, "pattern_coverage": 0.0,
         "section_completeness": 0.83, "red_flag_count": 1, "active_patterns": 0},
        {"label": "After Edit 1", "uncited_ratio": 0.18, "pattern_coverage": 0.33,
         "section_completeness": 1.0, "red_flag_count": 2, "active_patterns": 1},
        {"label": "After Edit 2", "uncited_ratio": 0.12, "pattern_coverage": 0.67,
         "section_completeness": 1.0, "red_flag_count": 3, "active_patterns": 2},
        {"label": "After Edit 3", "uncited_ratio": 0.08, "pattern_coverage": 1.0,
         "section_completeness": 1.0, "red_flag_count": 4, "active_patterns": 3},
    ]
    print(build_eval_table(example_metrics))