kvl/report.py

"""Generate a Markdown Knowledge Value Report from evaluation results."""

from __future__ import annotations
from datetime import datetime
from kvl.config import DIMENSION_META, KVS_CLASSIFICATION, MODELS


def _bar(score: int, width: int = 20) -> str:
    filled = round(score / 100 * width)
    return "█" * filled + "░" * (width - filled)


def _sensitivity_badge(level: str) -> str:
    icons = {"High": "🔴", "Moderate": "🟡", "Low": "🟢"}
    return f"{icons.get(level, '')} {level}"


def generate(doc_title: str, kvs_result: dict, module_results: dict, meta: dict | None = None) -> str:
    now = datetime.now().strftime("%Y-%m-%d %H:%M")
    kvs = kvs_result["kvs"]
    classification = kvs_result["classification"]
    dim_scores = kvs_result["dimension_scores"]
    contributions = kvs_result["weighted_contributions"]
    recommendations = kvs_result["recommendations"]

    eval_date = (meta or {}).get("eval_date", now)
    framework_version = (meta or {}).get("framework_version", "KVL v0.1")

    lines = [
        "# Knowledge Value Report",
        "",
        f"**Document:** {doc_title}  ",
        f"**Evaluated:** {eval_date}  ",
        f"**Framework:** {framework_version}",
        "",
    ]

    # ── Model Metadata block ──────────────────────────────────────────────────
    lines += [
        "## Evaluation Models",
        "",
        "> ⚠️ **Score validity notice:** Knowledge Novelty and Generation Utility scores are",
        "> **model-relative** — they measure the marginal value of this document to the specific",
        "> AI models listed below. Scores will change if these models are updated or replaced.",
        "> Retrieval Utility and Demand Utility are largely model-independent.",
        "> Re-evaluate after any major model version change.",
        "",
        "| Role | Model | Dimensions |",
        "|---|---|---|",
        f"| Evaluation judge | {MODELS['judge']['display']} (`{MODELS['judge']['id']}`) | Novelty scoring, Grounding, Demand |",
        f"| Answer generation | {MODELS['worker']['display']} (`{MODELS['worker']['id']}`) | Closed-book QA, Baseline & RAG answers, Queries |",
        f"| Text embeddings | {MODELS['embedder']['display']} | Retrieval index, Semantic similarity |",
        "",
        "---",
        "",
    ]

    # ── KVS Summary ───────────────────────────────────────────────────────────
    lines += [
        "## Overall Knowledge Value Score",
        "",
        "```",
        f"  {kvs} / 100   —   {classification}",
        f"  {_bar(kvs)}",
        "```",
        "",
        "| Band | Range | Description |",
        "|---|---|---|",
    ]
    for threshold, label, desc in KVS_CLASSIFICATION:
        marker = " ◀ this document" if label == classification else ""
        lines.append(f"| **{label}** | {threshold}–{threshold+19 if threshold < 81 else 100} | {desc}{marker} |")
    lines += ["", "---", ""]

    # ── Dimension table ───────────────────────────────────────────────────────
    lines += [
        "## Dimension Scores",
        "",
        "| Dimension | Score | Bar | Weight | Contribution | Model Sensitivity |",
        "|---|---|---|---|---|---|",
    ]
    for key, dmeta in DIMENSION_META.items():
        sc = dim_scores[key]
        contrib = contributions[key]
        sens = _sensitivity_badge(dmeta["model_sensitivity"])
        lines.append(
            f"| {dmeta['name']} | {sc}/100 | `{_bar(sc, 10)}` "
            f"| {int(dmeta['weight']*100)}% | {contrib} | {sens} |"
        )
    lines += ["", "---", ""]

    # ── Per-dimension detail ──────────────────────────────────────────────────
    section_order = ["novelty", "retrieval", "generation", "attribution", "demand"]
    section_titles = {
        "novelty":     "Knowledge Novelty",
        "retrieval":   "Retrieval Utility",
        "generation":  "Generation Utility",
        "attribution": "Attribution & Grounding",
        "demand":      "Demand Utility",
    }

    for key in section_order:
        dmeta = DIMENSION_META[key]
        sc = dim_scores[key]
        sens = _sensitivity_badge(dmeta["model_sensitivity"])
        lines += [
            f"## {section_titles[key]}",
            "",
            f"**Score: {sc}/100** &nbsp;|&nbsp; Model sensitivity: {sens}",
            "",
            f"_{dmeta['description']}_",
            "",
            f"**How measured:** {dmeta['how_measured']}",
            "",
            f"**Models used:** {', '.join(dmeta['models_used'])}",
            "",
            f"**Model sensitivity note:** {dmeta['sensitivity_note']}",
            "",
            f"{module_results.get(key, {}).get('summary', '')}",
            "",
        ]

        if key == "novelty":
            details = module_results.get("novelty", {}).get("details", [])
            if details:
                lines += [
                    f"<details><summary>Claim-by-claim breakdown ({len(details)} claims)</summary>",
                    "",
                    "| Claim | Known Score | Notes |",
                    "|---|---|---|",
                ]
                for d in details:
                    lines.append(f"| {d['claim'][:80]} | {d['known_score']:.2f} | {d['reason'][:60]} |")
                lines += ["", "</details>", ""]

        elif key == "retrieval":
            details = module_results.get("retrieval", {}).get("details", [])
            if details:
                lines += [
                    f"<details><summary>Query-by-query results ({len(details)} queries)</summary>",
                    "",
                    "| Query | Recall@3 | MRR |",
                    "|---|---|---|",
                ]
                for d in details:
                    lines.append(f"| {d['query'][:80]} | {d['recall_at_3']:.2f} | {d['reciprocal_rank']:.2f} |")
                lines += ["", "</details>", ""]

        elif key == "generation":
            details = module_results.get("generation", {}).get("details", [])
            if details:
                lines += [
                    f"<details><summary>Question-by-question results ({len(details)} questions)</summary>",
                    "",
                    "| Question | Improvement | Reason |",
                    "|---|---|---|",
                ]
                for d in details:
                    lines.append(f"| {d['question'][:80]} | {d['improvement']}/100 | {d['reason'][:60]} |")
                lines += ["", "</details>", ""]

        elif key == "attribution":
            details = module_results.get("attribution", {}).get("details", [])
            if details:
                lines += [
                    f"<details><summary>Per-answer grounding ({len(details)} answers)</summary>",
                    "",
                    "| Question | Grounding % | Hallucination | Semantic Sim |",
                    "|---|---|---|---|",
                ]
                for d in details:
                    halluc = "Yes ⚠️" if d.get("hallucination_detected") else "No"
                    lines.append(
                        f"| {d.get('question','')[:70]} "
                        f"| {round(d['grounding_fraction']*100)}% "
                        f"| {halluc} "
                        f"| {d['semantic_similarity']} |"
                    )
                lines += ["", "</details>", ""]

        elif key == "demand":
            topics = module_results.get("demand", {}).get("topics", [])
            if topics:
                lines += [
                    "| Topic | Query Freq (1-10) | Priority Domain | Unmet Need |",
                    "|---|---|---|---|",
                ]
                for t in topics:
                    lines.append(
                        f"| {t.get('topic','')[:50]} "
                        f"| {t.get('query_frequency','-')}/10 "
                        f"| {'Yes' if t.get('priority_domain') else 'No'} "
                        f"| {'Yes' if t.get('unmet_need') else 'No'} |"
                    )
                lines.append("")

        lines += ["---", ""]

    # ── Recommendations ───────────────────────────────────────────────────────
    lines += ["## Recommended Actions", ""]
    for rec in recommendations:
        lines.append(f"- {rec}")

    lines += [
        "",
        "---",
        "",
        f"*Generated by {framework_version} · Evaluated {eval_date}*  ",
        f"*Judge: {MODELS['judge']['display']} · Worker: {MODELS['worker']['display']} · Embeddings: {MODELS['embedder']['display']}*  ",
        "*Scores are model-relative. Re-evaluate after major model updates.*",
    ]

    return "\n".join(lines)