kvl/config.py

"""Centralized model configuration and dimension metadata for KVL."""

from __future__ import annotations
from datetime import datetime

# ── Model identifiers ─────────────────────────────────────────────────────────

MODELS = {
    "judge": {
        "id":      "claude-sonnet-4-6",
        "display": "Claude Sonnet 4.6",
        "role":    "Evaluation judge — novelty scoring, grounding assessment, demand analysis",
        "modules": ["A", "D", "E"],
    },
    "worker": {
        "id":      "claude-haiku-4-5-20251001",
        "display": "Claude Haiku 4.5",
        "role":    "Closed-book QA, baseline & RAG answer generation, query generation",
        "modules": ["A", "B", "C"],
    },
    "embedder": {
        "id":      "sentence-transformers/all-MiniLM-L6-v2",
        "display": "all-MiniLM-L6-v2",
        "role":    "Text embeddings for retrieval index and semantic similarity",
        "modules": ["B", "D"],
    },
}


def model_meta(eval_date: str | None = None) -> dict:
    return {
        "judge":    MODELS["judge"],
        "worker":   MODELS["worker"],
        "embedder": MODELS["embedder"],
        "eval_date": eval_date or datetime.now().strftime("%Y-%m-%d %H:%M UTC"),
        "framework_version": "KVL v0.1",
    }


# ── Per-dimension metadata ────────────────────────────────────────────────────

DIMENSION_META = {
    "novelty": {
        "name":              "Knowledge Novelty",
        "weight":            0.30,
        "model_sensitivity": "High",
        "sensitivity_note":  (
            "Directly tied to the specific model evaluated. A document scoring 96/100 "
            "against Claude Sonnet may score significantly lower against a model trained "
            "on domain-specific corpora. Re-evaluate when the underlying model is updated."
        ),
        "description": (
            "Measures how much of the document's content is already embedded in the AI "
            "model's pre-training. A high novelty score means the model cannot reliably "
            "answer questions about this document from its training alone — the document "
            "contributes genuinely new knowledge to the system."
        ),
        "how_measured": (
            "12–15 specific factual claims are extracted from the document. For each claim, "
            "a question is generated and posed to the model **without** the document "
            "(closed-book). A judge model scores how well the model's answer matches the "
            "document's claim. **Novelty = 100 × (1 − fraction already known)**."
        ),
        "high_means": "Document contains knowledge rare or absent in the model's training data.",
        "low_means":  "Document content is widely represented in the model's training corpus.",
        "models_used": ["Claude Haiku 4.5 (closed-book QA)", "Claude Sonnet 4.6 (judge)"],
    },
    "retrieval": {
        "name":              "Retrieval Utility",
        "weight":            0.20,
        "model_sensitivity": "Low",
        "sensitivity_note":  (
            "The most model-independent dimension. Embeddings use a fixed local model "
            "(all-MiniLM-L6-v2); only test-query generation involves the LLM. "
            "Scores are stable across LLM versions."
        ),
        "description": (
            "Measures how effectively the document surfaces in response to relevant user "
            "queries in a retrieval-augmented system. High retrieval utility means the "
            "document's content consistently appears in top search results when users "
            "ask about its topics."
        ),
        "how_measured": (
            "The document is split into ~400-word chunks and embedded using "
            "sentence-transformers. 8 representative queries are generated from the document, "
            "and for each query the system measures whether the most relevant chunk appears "
            "in the top-3 results. **Score = weighted Recall@3 (60%) + MRR (40%)**."
        ),
        "high_means": "Well-structured content that maps cleanly to user search queries.",
        "low_means":  "Poor chunk boundaries, dense prose, or missing structure — can be improved through document restructuring.",
        "models_used": ["all-MiniLM-L6-v2 (embeddings)", "Claude Haiku 4.5 (query generation)"],
    },
    "generation": {
        "name":              "Generation Utility",
        "weight":            0.25,
        "model_sensitivity": "High",
        "sensitivity_note":  (
            "Highly model-relative. A model with stronger prior knowledge will show less "
            "improvement from the document, yielding a lower generation utility score "
            "even if the document is objectively high quality. Re-evaluate when the model changes."
        ),
        "description": (
            "Measures the uplift in AI-generated answer quality when the document is "
            "provided as context (RAG) versus answering from the model's prior knowledge "
            "alone. This directly quantifies how much the document improves what the "
            "AI system can tell users."
        ),
        "how_measured": (
            "8 questions are derived from the document. Each is answered twice: once "
            "**without** the document (baseline) and once **with** the document as "
            "context (RAG). A judge model scores the improvement across accuracy, "
            "completeness, and specificity on a 0–100 scale."
        ),
        "high_means": "Document substantially improves AI answers — essential for advisory and QA applications.",
        "low_means":  "Model can already answer adequately without the document.",
        "models_used": ["Claude Haiku 4.5 (answer generation)", "Claude Sonnet 4.6 (judge)"],
    },
    "attribution": {
        "name":              "Attribution & Grounding",
        "weight":            0.15,
        "model_sensitivity": "Moderate",
        "sensitivity_note":  (
            "The LLM judge's grounding assessment varies across models. The semantic "
            "similarity component (sentence-transformers) is model-independent, providing "
            "a stable floor for the score."
        ),
        "description": (
            "Measures whether AI-generated answers are genuinely grounded in the document, "
            "rather than mixing in unverified claims or hallucinations. High grounding means "
            "the document's evidence is being faithfully used and answers are trustworthy "
            "for deployment in high-stakes domains."
        ),
        "how_measured": (
            "RAG answers from the Generation module are analysed by a judge model to "
            "identify which claims are traceable to specific document passages. Semantic "
            "similarity between the answer and document context provides a second signal. "
            "A score penalty is applied if hallucination is detected. "
            "**Score = 70% × LLM grounding fraction + 30% × semantic similarity − hallucination penalty**."
        ),
        "high_means": "Answers closely follow document — high trustworthiness for regulated or high-stakes deployment.",
        "low_means":  "Answers stray from document content — document may lack specificity or model over-relies on prior knowledge.",
        "models_used": ["Claude Sonnet 4.6 (grounding judge)", "all-MiniLM-L6-v2 (semantic similarity)"],
    },
    "demand": {
        "name":              "Demand Utility",
        "weight":            0.10,
        "model_sensitivity": "Low",
        "sensitivity_note":  (
            "Primarily reflects the document's subject matter rather than the model's "
            "knowledge state. Scores are relatively stable across LLM versions, though "
            "different models may assess domain importance differently."
        ),
        "description": (
            "Estimates how frequently the knowledge in this document would be needed by "
            "real users — considering topic popularity, domain priority (healthcare, climate, "
            "agriculture, policy), geographic relevance, and coverage of unmet information needs."
        ),
        "how_measured": (
            "5–8 topics are extracted from the document. For each, the model estimates "
            "query frequency (1–10), whether the domain is a priority area, and whether "
            "significant unmet information needs exist. A holistic demand score from the "
            "judge is blended with the topic-level estimates "
            "(40% topic average + 60% holistic score)."
        ),
        "high_means": "Broad audience or urgent unmet need — wide deployment and open access recommended.",
        "low_means":  "Narrow or specialised audience — targeted distribution channels are more appropriate.",
        "models_used": ["Claude Haiku 4.5 (topic extraction)", "Claude Sonnet 4.6 (demand assessment)"],
    },
}


SENSITIVITY_COLOR = {
    "High":     "#ef476f",
    "Moderate": "#f8961e",
    "Low":      "#06d6a0",
}

KVS_CLASSIFICATION = [
    (81, "Transformational Value", "Document adds exceptional, rare knowledge and dramatically improves AI system performance."),
    (61, "High Value",             "Document adds significant novel knowledge with strong retrieval and generation impact."),
    (41, "Moderate Value",         "Document offers meaningful contributions in at least some dimensions."),
    (21, "Incremental Value",      "Document provides marginal improvements; content may be partially redundant."),
    (0,  "Minimal Value",          "Little or no measurable improvement to AI systems from this document."),
]