"""Generate a Markdown Knowledge Value Report from evaluation results."""
from __future__ import annotations
from datetime import datetime
from kvl.config import DIMENSION_META, KVS_CLASSIFICATION, MODELS
def _bar(score: int, width: int = 20) -> str:
filled = round(score / 100 * width)
return "โ" * filled + "โ" * (width - filled)
def _sensitivity_badge(level: str) -> str:
icons = {"High": "๐ด", "Moderate": "๐ก", "Low": "๐ข"}
return f"{icons.get(level, '')} {level}"
def generate(doc_title: str, kvs_result: dict, module_results: dict, meta: dict | None = None) -> str:
now = datetime.now().strftime("%Y-%m-%d %H:%M")
kvs = kvs_result["kvs"]
classification = kvs_result["classification"]
dim_scores = kvs_result["dimension_scores"]
contributions = kvs_result["weighted_contributions"]
recommendations = kvs_result["recommendations"]
eval_date = (meta or {}).get("eval_date", now)
framework_version = (meta or {}).get("framework_version", "KVL v0.1")
lines = [
"# Knowledge Value Report",
"",
f"**Document:** {doc_title} ",
f"**Evaluated:** {eval_date} ",
f"**Framework:** {framework_version}",
"",
]
# โโ Model Metadata block โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
lines += [
"## Evaluation Models",
"",
"> โ ๏ธ **Score validity notice:** Knowledge Novelty and Generation Utility scores are",
"> **model-relative** โ they measure the marginal value of this document to the specific",
"> AI models listed below. Scores will change if these models are updated or replaced.",
"> Retrieval Utility and Demand Utility are largely model-independent.",
"> Re-evaluate after any major model version change.",
"",
"| Role | Model | Dimensions |",
"|---|---|---|",
f"| Evaluation judge | {MODELS['judge']['display']} (`{MODELS['judge']['id']}`) | Novelty scoring, Grounding, Demand |",
f"| Answer generation | {MODELS['worker']['display']} (`{MODELS['worker']['id']}`) | Closed-book QA, Baseline & RAG answers, Queries |",
f"| Text embeddings | {MODELS['embedder']['display']} | Retrieval index, Semantic similarity |",
"",
"---",
"",
]
# โโ KVS Summary โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
lines += [
"## Overall Knowledge Value Score",
"",
"```",
f" {kvs} / 100 โ {classification}",
f" {_bar(kvs)}",
"```",
"",
"| Band | Range | Description |",
"|---|---|---|",
]
for threshold, label, desc in KVS_CLASSIFICATION:
marker = " โ this document" if label == classification else ""
lines.append(f"| **{label}** | {threshold}โ{threshold+19 if threshold < 81 else 100} | {desc}{marker} |")
lines += ["", "---", ""]
# โโ Dimension table โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
lines += [
"## Dimension Scores",
"",
"| Dimension | Score | Bar | Weight | Contribution | Model Sensitivity |",
"|---|---|---|---|---|---|",
]
for key, dmeta in DIMENSION_META.items():
sc = dim_scores[key]
contrib = contributions[key]
sens = _sensitivity_badge(dmeta["model_sensitivity"])
lines.append(
f"| {dmeta['name']} | {sc}/100 | `{_bar(sc, 10)}` "
f"| {int(dmeta['weight']*100)}% | {contrib} | {sens} |"
)
lines += ["", "---", ""]
# โโ Per-dimension detail โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
section_order = ["novelty", "retrieval", "generation", "attribution", "demand"]
section_titles = {
"novelty": "Knowledge Novelty",
"retrieval": "Retrieval Utility",
"generation": "Generation Utility",
"attribution": "Attribution & Grounding",
"demand": "Demand Utility",
}
for key in section_order:
dmeta = DIMENSION_META[key]
sc = dim_scores[key]
sens = _sensitivity_badge(dmeta["model_sensitivity"])
lines += [
f"## {section_titles[key]}",
"",
f"**Score: {sc}/100** | Model sensitivity: {sens}",
"",
f"_{dmeta['description']}_",
"",
f"**How measured:** {dmeta['how_measured']}",
"",
f"**Models used:** {', '.join(dmeta['models_used'])}",
"",
f"**Model sensitivity note:** {dmeta['sensitivity_note']}",
"",
f"{module_results.get(key, {}).get('summary', '')}",
"",
]
if key == "novelty":
details = module_results.get("novelty", {}).get("details", [])
if details:
lines += [
f"Claim-by-claim breakdown ({len(details)} claims)
",
"",
"| Claim | Known Score | Notes |",
"|---|---|---|",
]
for d in details:
lines.append(f"| {d['claim'][:80]} | {d['known_score']:.2f} | {d['reason'][:60]} |")
lines += ["", "Query-by-query results ({len(details)} queries)
",
"",
"| Query | Recall@3 | MRR |",
"|---|---|---|",
]
for d in details:
lines.append(f"| {d['query'][:80]} | {d['recall_at_3']:.2f} | {d['reciprocal_rank']:.2f} |")
lines += ["", "Question-by-question results ({len(details)} questions)
",
"",
"| Question | Improvement | Reason |",
"|---|---|---|",
]
for d in details:
lines.append(f"| {d['question'][:80]} | {d['improvement']}/100 | {d['reason'][:60]} |")
lines += ["", "Per-answer grounding ({len(details)} answers)
",
"",
"| Question | Grounding % | Hallucination | Semantic Sim |",
"|---|---|---|---|",
]
for d in details:
halluc = "Yes โ ๏ธ" if d.get("hallucination_detected") else "No"
lines.append(
f"| {d.get('question','')[:70]} "
f"| {round(d['grounding_fraction']*100)}% "
f"| {halluc} "
f"| {d['semantic_similarity']} |"
)
lines += ["", "