rag_evaluation.py

"""Deterministic sentence-, citation-, and graph-level RAG evaluation."""

from __future__ import annotations

import re
from typing import Any

from tracernet.services.pathway_evidence import (
    build_default_matcher,
    canonical_formula,
    denies_transformation,
    extract_formula_mentions,
    normalize_text,
    parse_path,
    path_components,
)

# Instrument and method names are written as acronyms across this literature
# (XRD, XPS, FTIR, SEM, SIMS, XANES, EDX...). Detecting them by shape rather
# than by an enumerated list keeps the check open-ended: a technique nobody
# thought to list is still verified against the source.
_ACRONYM_RE = re.compile(r"(?<![A-Za-z])[A-Z]{2,6}(?![A-Za-z])")

# A paraphrase may reuse every entity of a snippet while asserting the
# opposite, so a negated or contrastive statement never qualifies as
# entity-grounded. Outright denial is defined once, in pathway_evidence
# (`denies_transformation`), and shared.
#
# The two patterns below extend that shared core for what this module judges: a
# single generated sentence rather than a whole literature chunk. Within one
# sentence a stray "is not" is likely to be the operative claim, so the strict
# reading is the safe one -- the opposite of what suits a long chunk.
_STATEMENT_NEGATION_RE = re.compile(
    r"\b(?:is|are|was|were|has|have|had|can|could|would|will)\s+not\b"
    r"|\b(?:isn't|aren't|wasn't|weren't|doesn't|didn't|cannot|can't)\b"
    r"|\bnever\b",
    flags=re.IGNORECASE,
)
# "A rather than B" does not deny the conversion, but it asserts an exclusion
# the source may not make, so grounding declines it as well.
_CONTRAST_RE = re.compile(
    r"\b(?:rather\s+than|instead\s+of|contrary\s+to|as\s+opposed\s+to)\b",
    flags=re.IGNORECASE,
)

_ENTITY_MATCHER = None


def _entity_resolver():
    """Shared chemical-name resolver (mineral name <-> formula)."""
    global _ENTITY_MATCHER
    if _ENTITY_MATCHER is None:
        import os

        _ENTITY_MATCHER = build_default_matcher(
            os.path.dirname(os.path.abspath(__file__))
        )
    return _ENTITY_MATCHER


def _grounding_entities(text: str) -> set[str]:
    """Chemical species and experimental terms that a claim must ground.

    Paraphrase is invisible here: only the *substance* of a sentence — which
    compounds, conditions and techniques it names — is extracted, so rewritten
    wording is fine while an invented species or technique is not.
    """
    entities: set[str] = set()
    normalized = normalize_text(_repair_formula_markup(text))
    folded = normalized.casefold()

    for mention in extract_formula_mentions(normalized):
        if mention.composition:
            entities.add("formula:" + mention.composition)

    resolver = _entity_resolver().resolver
    for name in resolver.by_name:
        if len(name) < 4:
            continue
        if re.search(rf"(?<!\w){re.escape(name)}(?!\w)", folded):
            records = resolver.by_name.get(name) or []
            formula = records[0].formula if records else ""
            entities.add("formula:" + (canonical_formula(formula) or name))

    # Method/instrument acronyms, by shape rather than by enumeration.
    for acronym in _ACRONYM_RE.findall(str(text or "")):
        entities.add("acronym:" + acronym.casefold())

    # Reaction conditions, taken from the project's own condition vocabulary
    # (CRN labels plus their injected aliases) rather than a list written here.
    # Wordings that the resolver treats as equivalent collapse to one concept,
    # so "light" grounds against a source that says "photoinduced".
    condition_aliases = _entity_resolver().condition_resolver.aliases
    for concept, wordings in condition_aliases.items():
        for term in {concept, *wordings}:
            term = str(term or "").strip().casefold()
            if len(term) < 2:
                continue
            if re.search(rf"(?<!\w){re.escape(term)}(?!\w)", folded):
                entities.add("condition:" + concept)
                break
    return entities

_STOPWORDS = {
    "a",
    "an",
    "and",
    "are",
    "as",
    "at",
    "be",
    "by",
    "can",
    "for",
    "from",
    "has",
    "have",
    "in",
    "into",
    "is",
    "it",
    "of",
    "on",
    "or",
    "that",
    "the",
    "their",
    "this",
    "to",
    "under",
    "via",
    "with",
    "within",
}

_SECTION_PREFIXES = (
    "provenance analysis report",
    "introduction",
    "chemical pathway",
    "evidence summary",
    "literature evidence",
    "references",
    "reliability assessment",
    "validation note:",
    "nodes:",
    "conditions:",
    "citation coverage:",
    "path consistency:",
    "evidence granularity:",
    "vector retrieval:",
    "collection size:",
    "retrieval basis:",
    "retrieved snippets:",
    "source count:",
    "composite evidence:",
    "unsupported claims:",
    "overall confidence:",
    "the crn records the graph relation",
    "no direct pathway-matching literature evidence was retrieved",
    "the literature index could not be queried",
    "the current evidence therefore does not establish",
    "the current evidence does not establish",
    "mechanistic interpretation:",
    "related literature context was retrieved",
    "no pathway-matching literature snippet is available",
    "no direct evidence for the complete pathway",
)


def _round_score(value: float | int | None) -> float:
    if value is None:
        return 0.0
    value = max(0.0, min(1.0, float(value)))
    return round(value, 3)


def _repair_formula_markup(value: Any) -> str:
    """Remove TeX wrappers, including wrappers damaged by earlier renderers.

    Some rendering pipelines flatten upright formula wrappers before
    evaluation, leaving command names attached to an otherwise valid formula.
    Display artefacts must not turn a supported chemical claim into a graph
    mismatch.
    """

    text = str(value or "")
    for _ in range(3):
        repaired = re.sub(
            r"\\(?:mathrm|mathbf|mathit|text|operatorname)\s*\{([^{}]*)\}",
            r"\1",
            text,
            flags=re.IGNORECASE,
        )
        if repaired == text:
            break
        text = repaired
    text = re.sub(
        r"(?<![A-Za-z])(?:mathrm|mathbf|mathit|operatorname)(?=[A-Z][a-z]?)",
        "",
        text,
        flags=re.IGNORECASE,
    )
    text = re.sub(r"_\{?(\d+)\}?", r"\1", text)
    return text.replace("{", "").replace("}", "")


def _normalize(text: Any) -> str:
    normalized = normalize_text(_repair_formula_markup(text))
    normalized = normalized.replace("\\xrightarrow", " arrow ")
    normalized = normalized.replace("\\rightarrow", " arrow ")
    normalized = normalized.replace("-->", " arrow ").replace("->", " arrow ")
    normalized = re.sub(r"[\[\]{}$]", " ", normalized)
    return re.sub(r"\s+", " ", normalized).strip().casefold()

def _tokens(text: Any) -> set[str]:
    norm = _normalize(text)
    toks = re.findall(r"[a-z0-9]+(?:-[a-z0-9]+)?", norm)
    return {t for t in toks if len(t) > 1 and t not in _STOPWORDS}


def _split_sentences(text: Any) -> list[str]:
    cleaned = str(text or "").replace("\r\n", "\n").replace("\r", "\n")
    cleaned = re.sub(r"\n{2,}", "\n", cleaned)
    chunks: list[str] = []
    for line in cleaned.splitlines():
        line = line.strip()
        if not line:
            continue
        chunks.extend(re.split(r"(?<=[.!?。！？])\s+", line))
    reattached: list[str] = []
    for chunk in chunks:
        leading = re.match(r"^((?:\[\d+\]\s*)+)(.*)$", chunk.strip())
        if leading and reattached:
            reattached[-1] = f"{reattached[-1]} {leading.group(1).strip()}"
            if leading.group(2).strip():
                reattached.append(leading.group(2).strip())
        else:
            reattached.append(chunk)
    out = []
    for chunk in reattached:
        sent = chunk.strip()
        if not sent:
            continue
        low = sent.strip("#* ").lower()
        if any(low.startswith(prefix) for prefix in _SECTION_PREFIXES):
            continue
        if (
            any(marker in sent for marker in ("--[", "->", "→", "\\xrightarrow"))
            and parse_path(sent)
        ):
            continue
        if len(_tokens(sent)) < 3 and not re.search(r"\[\d+\]", sent):
            continue
        out.append(sent)
    return out


def _path_terms(path_str: Any) -> list[str]:
    species, conditions = path_components(path_str, keep_unparsed=True)
    terms: list[str] = []
    for item in species + conditions:
        item = str(item or "").strip()
        if not item:
            continue
        terms.append(item)
        for sub in re.split(r"\s*[+;/]\s*", item):
            sub = sub.strip()
            if sub and sub != item:
                terms.append(sub)
    seen = set()
    ordered = []
    for term in terms:
        key = _normalize(term)
        if key and key not in seen:
            seen.add(key)
            ordered.append(term)
    return ordered


def _term_is_covered(term: str, text: Any) -> bool:
    norm_term = _normalize(term)
    norm_text = _normalize(text)
    if not norm_term or not norm_text:
        return False
    if norm_term in norm_text:
        return True
    term_tokens = _tokens(term)
    if not term_tokens:
        return False
    text_tokens = _tokens(text)
    overlap = len(term_tokens & text_tokens)
    if len(term_tokens) == 1:
        return overlap == 1
    return overlap / len(term_tokens) >= 0.5


def _citation_nums(sentence: str) -> list[int]:
    nums = []
    for raw in re.findall(r"\[(\d+)\]", sentence or ""):
        try:
            nums.append(int(raw))
        except Exception:
            pass
    return nums


def _overlap_score(sentence: str, snippet: str) -> float:
    sent_tokens = _tokens(sentence)
    snippet_tokens = _tokens(snippet)
    if not sent_tokens or not snippet_tokens:
        return 0.0
    if _normalize(sentence) in _normalize(snippet):
        return 1.0
    return min(
        1.0,
        len(sent_tokens & snippet_tokens) / max(1, min(len(sent_tokens), 18)),
    )




def split_into_statements(report: Any) -> list[str]:
    """Public sentence splitter for post-generation RAG evaluation."""

    return _split_sentences(report)


def _evidence_text(evidence: Any) -> str:
    if isinstance(evidence, dict):
        return re.sub(
            r"\s+",
            " ",
            str(evidence.get("snippet") or evidence.get("text") or evidence.get("content") or ""),
        ).strip()
    return re.sub(r"\s+", " ", str(evidence or "")).strip()


def _evidence_source(evidence: Any) -> str:
    if not isinstance(evidence, dict):
        return ""
    return re.sub(
        r"\s+",
        " ",
        str(evidence.get("source") or evidence.get("reference") or evidence.get("doi") or ""),
    ).strip()


def _is_direct_evidence_block(evidence: Any) -> bool:
    if not isinstance(evidence, dict):
        return False
    edge_matches = evidence.get("edge_matches")
    if isinstance(edge_matches, list) and edge_matches:
        return any(
            isinstance(match, dict) and match.get("verdict") == "direct"
            for match in edge_matches
        )
    status = str(evidence.get("evidence_status") or "").strip().casefold()
    return status == "direct"


def _statement_matches_direct_edge(
    statement: str,
    evidence: dict[str, Any],
) -> bool:
    edge_matches = evidence.get("edge_matches")
    if not isinstance(edge_matches, list) or not edge_matches:
        return _is_direct_evidence_block(evidence)
    for match in edge_matches:
        if not isinstance(match, dict) or match.get("verdict") != "direct":
            continue
        reactant = str(match.get("reactant") or "").strip()
        product = str(match.get("product") or "").strip()
        reactant_terms = [
            term
            for term in (reactant, str(match.get("reactant_span") or "").strip())
            if term
        ]
        product_terms = [
            term
            for term in (product, str(match.get("product_span") or "").strip())
            if term
        ]
        if (
            reactant_terms
            and product_terms
            and any(_term_is_covered(term, statement) for term in reactant_terms)
            and any(_term_is_covered(term, statement) for term in product_terms)
        ):
            return True
    return False


def _block_supports_statement(
    statement: str,
    evidence: dict[str, Any],
) -> bool:
    return (
        _is_direct_evidence_block(evidence)
        and _statement_matches_direct_edge(statement, evidence)
    )


# A statement that merely *restates* a cited snippet (mechanism, conditions,
# instrumentation, observed phases) is grounded even though it asserts no graph
# edge, so it cannot satisfy the reactant+product test above. Requiring a much
# higher lexical overlap keeps this from becoming a loophole: the sentence has
# to closely track the snippet it cites.
RESTATEMENT_THRESHOLD = 0.55


def match_restated_evidence(
    statement: str,
    retrieved_snippets: list[Any] | None,
    threshold: float = RESTATEMENT_THRESHOLD,
) -> dict[str, Any] | None:
    """Find a direct-evidence snippet that this statement closely restates."""

    best_block = None
    best_score = 0.0
    for block in _normalize_evidence_blocks(retrieved_snippets, None):
        if not _is_direct_evidence_block(block):
            continue
        snippet = _evidence_text(block)
        if not snippet:
            continue
        score = _overlap_score(statement, snippet)
        if score > best_score:
            best_score = score
            best_block = block
    if best_block is None or best_score < threshold:
        return None
    out = dict(best_block)
    out["match_score"] = _round_score(best_score)
    return out


def match_entity_grounded_evidence(
    statement: str,
    retrieved_snippets: list[Any] | None,
) -> dict[str, Any] | None:
    """Find a cited snippet that contains every entity the statement asserts.

    This is the paraphrase-tolerant tier. Lexical overlap is deliberately not
    consulted: a sentence written in genuinely new words is accepted as long as
    every chemical species, condition and technique it names is present in the
    snippet, and it does not negate that snippet. A sentence that introduces an
    entity the source never mentions is rejected, which is what keeps invented
    mechanisms out.
    """
    claimed = _grounding_entities(statement)
    if not claimed:
        return None
    if (
        denies_transformation(statement)
        or _STATEMENT_NEGATION_RE.search(statement or "")
        or _CONTRAST_RE.search(statement or "")
    ):
        return None

    best_block = None
    best_ratio = 0.0
    for block in _normalize_evidence_blocks(retrieved_snippets, None):
        if not _is_direct_evidence_block(block):
            continue
        snippet = _evidence_text(block)
        if not snippet:
            continue
        available = _grounding_entities(snippet)
        if not claimed <= available:
            continue
        ratio = len(claimed) / max(1, len(available))
        if best_block is None or ratio > best_ratio:
            best_block = block
            best_ratio = ratio
    if best_block is None:
        return None
    out = dict(best_block)
    out["match_score"] = _round_score(min(1.0, 0.5 + best_ratio / 2))
    out["grounding"] = "entity"
    return out


def _normalize_evidence_blocks(retrieved_snippets: list[Any] | None, references: list[str] | None) -> list[dict[str, Any]]:
    references = references or []
    blocks: list[dict[str, Any]] = []
    for idx, item in enumerate(retrieved_snippets or [], 1):
        if isinstance(item, dict):
            block = dict(item)
            block.setdefault("index", idx)
            block.setdefault("reference_index", block.get("index", idx))
            if "snippet" not in block and "text" in block:
                block["snippet"] = block.get("text")
            ref_index = block.get("reference_index")
            if (
                not block.get("source")
                and isinstance(ref_index, int)
                and 1 <= ref_index <= len(references)
            ):
                block["source"] = references[ref_index - 1]
        else:
            block = {
                "index": idx,
                "reference_index": idx,
                "snippet": str(item or ""),
                "source": references[idx - 1] if idx <= len(references) else "",
                "evidence_status": "unclassified",
            }
        source = _evidence_source(block)
        dois = re.findall(r"10\.\d{4,9}/[^\s,;)\]]+", source, flags=re.I)
        if dois and not block.get("doi"):
            block["doi"] = dois[0].rstrip(".")
        blocks.append(block)
    return blocks


def match_supporting_evidence(statement: str, retrieved_snippets: list[Any] | None, threshold: float = 0.18) -> dict[str, Any] | None:
    """Find the best retrieved snippet for one generated statement."""

    best_block = None
    best_score = 0.0
    for block in _normalize_evidence_blocks(retrieved_snippets, None):
        if not _block_supports_statement(statement, block):
            continue
        snippet = _evidence_text(block)
        if not snippet:
            continue
        score = _overlap_score(statement, snippet)
        if score > best_score:
            best_score = score
            best_block = block
    if best_block is None or best_score < threshold:
        return None
    out = dict(best_block)
    out["match_score"] = _round_score(best_score)
    return out


def _pathway_terms(pathway_graph: Any) -> list[str]:
    if isinstance(pathway_graph, dict):
        terms = list(pathway_graph.get("path_terms") or [])
        path_str = pathway_graph.get("path_str") or ""
        if path_str:
            terms.extend(_path_terms(path_str))
        graph_refs = pathway_graph.get("graph_refs") or {}
        if isinstance(graph_refs, dict):
            terms.extend(graph_refs.get("node_ids") or [])
            terms.extend(graph_refs.get("edge_ids") or [])
        terms.extend(pathway_graph.get("node_ids") or [])
        terms.extend(pathway_graph.get("edge_ids") or [])
    elif isinstance(pathway_graph, str):
        terms = _path_terms(pathway_graph)
    else:
        terms = []

    seen = set()
    ordered = []
    for term in terms:
        key = _normalize(term)
        if key and key not in seen:
            seen.add(key)
            ordered.append(str(term))
    return ordered


def _graph_node_ids(pathway_graph: Any) -> list[str]:
    if not isinstance(pathway_graph, dict):
        return []
    raw = list(pathway_graph.get("node_ids") or [])
    refs = pathway_graph.get("graph_refs") or {}
    if isinstance(refs, dict):
        raw.extend(refs.get("node_ids") or [])

    seen: set[str] = set()
    out: list[str] = []
    for item in raw:
        value = str(item or "").strip()
        key = _normalize(value)
        if key and key not in seen:
            seen.add(key)
            out.append(value)
    return out


def _ordered_path_species(pathway_graph: Any) -> list[str]:
    if isinstance(pathway_graph, str):
        path_str = pathway_graph
    elif isinstance(pathway_graph, dict):
        path_str = pathway_graph.get("path_str") or ""
    else:
        path_str = ""

    species, _conditions = path_components(path_str, keep_unparsed=True)
    if len(species) >= 2:
        return species
    if not isinstance(pathway_graph, dict):
        return []

    for key in ("path_species", "species_path", "path_nodes", "ordered_nodes"):
        raw = pathway_graph.get(key)
        if not isinstance(raw, (list, tuple)):
            continue
        labels: list[str] = []
        for node in raw:
            if isinstance(node, dict):
                node = (
                    node.get("formula")
                    or node.get("label")
                    or node.get("name")
                    or node.get("id")
                    or node.get("node_id")
                )
            value = str(node or "").strip()
            if value:
                labels.append(value)
        if len(labels) >= 2:
            return labels
    return []


def _explicit_graph_edges(pathway_graph: Any) -> list[tuple[str, str]]:
    if not isinstance(pathway_graph, dict):
        return []
    raw_edges = list(pathway_graph.get("path_edges") or [])
    raw_edges.extend(pathway_graph.get("edges") or [])
    refs = pathway_graph.get("graph_refs") or {}
    if isinstance(refs, dict):
        raw_edges.extend(refs.get("edges") or [])

    pairs: list[tuple[str, str]] = []
    for edge in raw_edges:
        source = target = ""
        if isinstance(edge, dict):
            source = edge.get("source") or edge.get("from") or edge.get("start") or ""
            target = edge.get("target") or edge.get("to") or edge.get("end") or ""
        elif isinstance(edge, (list, tuple)) and len(edge) >= 2:
            source, target = edge[0], edge[1]
        elif isinstance(edge, str):
            parsed, _conditions = path_components(edge, keep_unparsed=True)
            if len(parsed) == 2:
                source, target = parsed
        source = str(source or "").strip()
        target = str(target or "").strip()
        if source and target:
            pairs.append((source, target))
    return pairs


def _graph_alignment_details(
    statement: str,
    pathway_graph: Any,
    evidence: dict[str, Any] | None = None,
) -> dict[str, Any]:
    """Require an edge or a validated start-to-final pathway relation."""

    species = _ordered_path_species(pathway_graph)
    if len(species) >= 2:
        endpoint_matches = [
            match
            for match in ((evidence or {}).get("edge_matches") or [])
            if (
                isinstance(match, dict)
                and match.get("verdict") == "direct"
                and match.get("evidence_scope") == "pathway_endpoint"
                and _normalize(match.get("reactant")) == _normalize(species[0])
                and _normalize(match.get("product")) == _normalize(species[-1])
            )
        ]
        if endpoint_matches and _statement_matches_direct_edge(
            statement,
            {"edge_matches": endpoint_matches},
        ):
            return {
                "aligned": True,
                "method": "validated_pathway_endpoints",
                "matched_species_pairs": [{
                    "source": species[0],
                    "target": species[-1],
                }],
                "required_species_pairs": [{
                    "source": species[0],
                    "target": species[-1],
                }],
                "matched_node_ids": [],
            }
        pairs = list(zip(species, species[1:]))
        matched = [
            {"source": source, "target": target}
            for source, target in pairs
            if _term_is_covered(source, statement) and _term_is_covered(target, statement)
        ]
        return {
            "aligned": bool(matched),
            "method": "adjacent_species_pair",
            "matched_species_pairs": matched,
            "required_species_pairs": [
                {"source": source, "target": target} for source, target in pairs
            ],
            "matched_node_ids": [],
        }

    edges = _explicit_graph_edges(pathway_graph)
    if edges:
        matched = [
            {"source": source, "target": target}
            for source, target in edges
            if _term_is_covered(source, statement) and _term_is_covered(target, statement)
        ]
        return {
            "aligned": bool(matched),
            "method": "explicit_graph_edge",
            "matched_species_pairs": matched,
            "required_species_pairs": [
                {"source": source, "target": target} for source, target in edges
            ],
            "matched_node_ids": [],
        }

    node_ids = _graph_node_ids(pathway_graph)
    if node_ids:
        matched_nodes = [node for node in node_ids if _term_is_covered(node, statement)]
        return {
            "aligned": bool(matched_nodes),
            "method": "node_id_fallback",
            "matched_species_pairs": [],
            "required_species_pairs": [],
            "matched_node_ids": matched_nodes,
        }

    terms = _pathway_terms(pathway_graph)
    matched_terms = [term for term in terms if _term_is_covered(term, statement)]
    return {
        "aligned": bool(matched_terms),
        "method": "term_fallback" if terms else "no_graph_terms",
        "matched_species_pairs": [],
        "required_species_pairs": [],
        "matched_node_ids": matched_terms,
    }




def check_citation_support(
    statement: str,
    evidence: dict[str, Any] | None,
    references: list[str] | None,
    retrieved_snippets: list[Any] | None = None,
) -> bool | None:
    cited_nums = _citation_nums(statement)
    if not cited_nums:
        return None
    references = references or []
    blocks = _normalize_evidence_blocks(retrieved_snippets, references)
    cited_blocks = []
    for n in cited_nums:
        matching = [
            block
            for block in blocks
            if block.get("reference_index") == n
        ]
        if matching:
            cited_blocks.extend(matching)
        elif 1 <= n <= len(references):
            cited_blocks.append({
                "index": n,
                "reference_index": n,
                "source": references[n - 1],
                "snippet": "",
                "evidence_status": "metadata_only",
            })
    if not cited_blocks:
        return False
    cited_snippet_blocks = [block for block in cited_blocks if _evidence_text(block)]
    if not cited_snippet_blocks:
        return None
    direct_blocks = [
        block for block in cited_snippet_blocks
        if _block_supports_statement(statement, block)
    ]
    if direct_blocks:
        return any(
            _overlap_score(statement, _evidence_text(block)) >= 0.18
            for block in direct_blocks
        )
    # The citation may instead back a close restatement of a direct-evidence
    # snippet rather than an edge claim; that is still a verified citation.
    return any(
        _is_direct_evidence_block(block)
        and _overlap_score(statement, _evidence_text(block)) >= RESTATEMENT_THRESHOLD
        for block in cited_snippet_blocks
    ) or match_entity_grounded_evidence(statement, cited_snippet_blocks) is not None


def _is_inference_statement(statement: str) -> bool:
    low = str(statement or "").lower()
    inference_markers = (
        "(inferred)",
        "inferred",
        "likely",
        "suggest",
        "therefore",
        "displayed reaction network",
        "traced pathway",
        "reaction network",
        "pathway condition",
        "conservation implication",
        "represented by",
    )
    return any(marker in low for marker in inference_markers)


def _source_traceability_assessment(
    evidence_blocks: list[dict[str, Any]],
    references: list[str],
) -> dict[str, Any]:
    """Score whether evidence can be traced and independently inspected.

    This is deliberately a documentation-quality score, not a journal-prestige
    score.  It rewards an identifiable publication, a DOI, an inspectable
    snippet, and an explicit evidence classification.
    """

    grouped: dict[int, list[dict[str, Any]]] = {}
    for block in evidence_blocks:
        try:
            reference_index = int(block.get("reference_index") or 0)
        except (TypeError, ValueError):
            reference_index = 0
        if reference_index <= 0:
            try:
                reference_index = int(block.get("index") or len(grouped) + 1)
            except (TypeError, ValueError):
                reference_index = len(grouped) + 1
        grouped.setdefault(reference_index, []).append(block)

    details: list[dict[str, Any]] = []
    for reference_index, blocks in sorted(grouped.items()):
        reference = (
            str(references[reference_index - 1]).strip()
            if 1 <= reference_index <= len(references)
            else ""
        )
        source_text = " ".join(
            part
            for part in [
                reference,
                *(_evidence_source(block) for block in blocks),
            ]
            if part
        )
        has_doi = bool(
            re.search(r"10\.\d{4,9}/[^\s,;)\]]+", source_text, flags=re.I)
            or any(block.get("doi") for block in blocks)
        )
        has_year = bool(re.search(r"\b(?:19|20)\d{2}\b", source_text))
        has_structured_citation = bool(
            re.search(
                r"\[(?:J|M|C|R|D)(?:/OL)?\]|\b(?:journal|proceedings|"
                r"transactions|review|letters)\b",
                source_text,
                flags=re.I,
            )
        )
        has_snippet = any(bool(_evidence_text(block)) for block in blocks)
        has_direct_classification = any(
            _is_direct_evidence_block(block) for block in blocks
        )
        score = _round_score(
            (0.25 if has_doi else 0.0)
            + (0.15 if has_year else 0.0)
            + (0.15 if has_structured_citation else 0.0)
            + (0.25 if has_snippet else 0.0)
            + (0.20 if has_direct_classification else 0.0)
        )
        level = "high" if score >= 0.8 else "moderate" if score >= 0.55 else "limited"
        details.append({
            "reference_index": reference_index,
            "source": reference or _evidence_source(blocks[0]),
            "score": score,
            "level": level,
            "has_doi": has_doi,
            "has_year": has_year,
            "has_structured_citation": has_structured_citation,
            "has_snippet": has_snippet,
            "has_direct_classification": has_direct_classification,
        })

    overall_score = _round_score(
        sum(item["score"] for item in details) / len(details)
        if details
        else 0.0
    )
    overall_level = (
        "high"
        if overall_score >= 0.8
        else "moderate"
        if overall_score >= 0.55
        else "limited"
        if details
        else "unavailable"
    )
    return {
        "score": overall_score,
        "level": overall_level,
        "details": details,
        "basis": (
            "Bibliographic traceability and evidence completeness; "
            "not journal prestige or independent replication."
        ),
    }


def _pathway_evidence_coverage(
    pathway_graph: Any,
    evidence_blocks: list[dict[str, Any]],
) -> tuple[float, bool]:
    edges = parse_path(
        pathway_graph.get("path_str") if isinstance(pathway_graph, dict) else pathway_graph
    )
    if not edges:
        return 0.0, False

    direct_edge_indexes: set[int] = set()
    endpoint_supported = False
    for block in evidence_blocks:
        for match in block.get("edge_matches") or []:
            if not isinstance(match, dict) or match.get("verdict") != "direct":
                continue
            if match.get("evidence_scope") == "pathway_endpoint":
                endpoint_supported = True
                continue
            try:
                edge_index = int(match.get("edge_index") or 0)
            except (TypeError, ValueError):
                edge_index = 0
            if 1 <= edge_index <= len(edges):
                direct_edge_indexes.add(edge_index)
                continue
            for edge in edges:
                if (
                    _normalize(match.get("reactant")) == _normalize(edge.reactant)
                    and _normalize(match.get("product")) == _normalize(edge.product)
                ):
                    direct_edge_indexes.add(edge.index)
                    break
    if len(edges) == 1 and direct_edge_indexes:
        endpoint_supported = True
    return _round_score(len(direct_edge_indexes) / len(edges)), endpoint_supported


def _overall_reliability_assessment(
    *,
    total: int,
    snippet_count: int,
    evidence_coverage: float,
    citation_accuracy: float | None,
    citation_checked: int,
    citation_unverifiable_count: int,
    graph_alignment: float,
    graph_alignment_applicable_count: int,
    source_quality_score: float,
    edge_evidence_coverage: float,
    endpoint_conversion_supported: bool,
    unsupported_count: int,
) -> dict[str, Any]:
    citation_component = (
        float(citation_accuracy)
        if citation_accuracy is not None
        else 0.0
    )
    graph_component = (
        graph_alignment if graph_alignment_applicable_count else 1.0
    )
    score = _round_score(
        0.35 * evidence_coverage
        + 0.20 * citation_component
        + 0.15 * graph_component
        + 0.15 * source_quality_score
        + 0.15 * edge_evidence_coverage
    )
    reasons: list[str] = []

    if total == 0:
        level = "not_evaluable"
        reasons.append("No evaluable factual statements were found.")
    elif snippet_count == 0:
        level = "insufficient_evidence"
        reasons.append("No snippet-level literature evidence was available.")
    else:
        failed_citations = max(0, citation_checked - int(round(
            citation_component * citation_checked
        )))
        reliable = bool(
            evidence_coverage >= 0.8
            and (
                graph_alignment_applicable_count == 0
                or graph_alignment >= 0.8
            )
            and source_quality_score >= 0.55
            and citation_checked > 0
            and citation_accuracy is not None
            and citation_accuracy >= 0.8
            and citation_unverifiable_count == 0
            and failed_citations == 0
            and unsupported_count == 0
            and edge_evidence_coverage >= 0.8
        )
        partially_reliable = bool(
            evidence_coverage >= 0.5
            and (
                graph_alignment_applicable_count == 0
                or graph_alignment >= 0.5
            )
            and source_quality_score >= 0.4
            and failed_citations == 0
        )
        if reliable:
            level = "reliable"
        elif partially_reliable:
            level = "partially_reliable"
        else:
            level = "insufficient_evidence"

        # Each reason names the measurement that fell short and the count behind
        # it. "Generated claims are not fully aligned with the selected path"
        # told a reader that something was wrong without saying what, how much,
        # or where to look -- which is the one thing a reliability conclusion has
        # to do.
        if unsupported_count:
            reasons.append(
                f"Statement support: {unsupported_count} of {total} generated "
                "statement(s) are not backed by any retrieved snippet."
            )
        if citation_checked == 0:
            reasons.append(
                "Citation verification: no cited statement could be checked "
                "against a linked snippet, so citation accuracy is unmeasured."
            )
        elif citation_accuracy is not None and citation_accuracy < 1:
            wrong = max(
                0, citation_checked - int(round(citation_component * citation_checked))
            )
            reasons.append(
                f"Citation verification: {wrong} of {citation_checked} checked "
                "citation(s) do not support the claim they are attached to."
            )
        if citation_unverifiable_count:
            reasons.append(
                f"Citation verification: {citation_unverifiable_count} "
                "citation(s) have no linked snippet to verify against."
            )
        if graph_alignment_applicable_count and graph_alignment < 0.8:
            aligned = int(round(graph_alignment * graph_alignment_applicable_count))
            reasons.append(
                f"Graph alignment: {aligned} of {graph_alignment_applicable_count} "
                f"path-related statement(s) match the selected path "
                f"({round(graph_alignment * 100)}%). The remainder describe "
                "species, conditions or steps that the selected path does not "
                "encode -- most often a condition reported by the literature "
                "that differs from the graph-encoded condition label."
            )
        if source_quality_score < 0.55:
            reasons.append(
                "Source traceability: bibliographic completeness scores "
                f"{round(source_quality_score * 100)}%; one or more references "
                "lack a DOI, year, or inspectable snippet."
            )
        if edge_evidence_coverage < 0.8:
            covered = round(edge_evidence_coverage * 100)
            if endpoint_conversion_supported:
                reasons.append(
                    f"Edge evidence coverage: {covered}%. The overall endpoint "
                    "conversion is supported by retrieved literature, but at "
                    "least one intermediate graph edge has no direct passage of "
                    "its own and remains graph-derived."
                )
            else:
                reasons.append(
                    f"Edge evidence coverage: {covered}%. One or more graph "
                    "edges have no direct literature passage establishing them."
                )

    summaries = {
        "reliable": (
            "Reliable: generated claims are supported by retrieved snippets, "
            "their citations are verified, and they align with the selected path."
        ),
        "partially_reliable": (
            # The dimensions that fell short are listed underneath, so the
            # summary points at them rather than saying "at least one", which
            # left the reader to guess which one and why.
            "Partially reliable: the retrieved literature supports the report's "
            "claims, but the dimensions listed below did not reach the "
            "threshold for a fully reliable rating."
        ),
        "insufficient_evidence": (
            "Insufficient evidence: the retrieved material does not support a "
            "reliable report-level conclusion."
        ),
        "not_evaluable": (
            "Not evaluable: no factual statements were available for assessment."
        ),
    }
    return {
        "score": score,
        "level": level,
        "summary": summaries[level],
        "reasons": reasons,
    }




def evaluate_rag_report(
    report: Any,
    retrieved_snippets: list[Any] | None,
    pathway_graph: Any,
    references: list[str] | None,
) -> dict[str, Any]:
    """Evaluate a generated RAG report at sentence level after LLM generation."""

    statements = split_into_statements(report)
    references = references or []
    evidence_blocks = _normalize_evidence_blocks(retrieved_snippets, references)

    results = []
    supported_count = 0
    direct_supported_count = 0
    restated_supported_count = 0
    unsupported_count = 0
    unsupported_by_both_count = 0
    any_supported_count = 0
    graph_aligned_count = 0
    graph_applicable_count = 0
    graph_derived_count = 0
    inference_count = 0
    citation_checked = 0
    citation_correct_count = 0
    citation_unverifiable_count = 0

    for idx, statement in enumerate(statements, 1):
        evidence = match_supporting_evidence(statement, evidence_blocks)
        graph_details = _graph_alignment_details(
            statement,
            pathway_graph,
            evidence=evidence,
        )
        graph_status = bool(graph_details["aligned"])
        explicit_inference = _is_inference_statement(statement)
        direct_evidence_status = evidence is not None and not explicit_inference
        # Secondary tier: the statement asserts no graph edge but closely
        # restates a cited direct-evidence snippet (mechanism, conditions,
        # instrumentation, observed phases). Grounded, so it is reported —
        # but kept distinct from edge-level direct evidence in the metrics.
        restated_evidence = None
        if not direct_evidence_status and not explicit_inference:
            restated_evidence = match_restated_evidence(statement, evidence_blocks)
            if restated_evidence is None:
                # Paraphrase-tolerant tier: judge the substance (which species,
                # conditions and techniques are asserted) rather than the
                # wording, so a genuinely rewritten synthesis is not discarded
                # while an invented entity still is.
                restated_evidence = match_entity_grounded_evidence(
                    statement, evidence_blocks
                )
        restated_status = restated_evidence is not None
        grounded_evidence_status = direct_evidence_status or restated_status
        graph_applicable_status = bool(
            direct_evidence_status or explicit_inference or graph_status
        )
        if direct_evidence_status:
            statement_type = "direct_evidence"
        elif restated_status:
            statement_type = "snippet_restatement"
        elif graph_status:
            statement_type = "graph_inference"
        else:
            statement_type = "unsupported"

        # Both an edge claim and a close, entity-grounded restatement are
        # supported by retrieved text.  They remain separate categories so the
        # report never presents mechanism prose as direct evidence for a graph
        # edge.  Graph alignment is reported separately and cannot inflate
        # snippet-evidence coverage.
        support_status = grounded_evidence_status
        any_support_status = grounded_evidence_status or graph_status
        graph_derived_status = graph_status and not grounded_evidence_status
        unsupported_status = not grounded_evidence_status
        unsupported_by_both_status = not any_support_status
        citation_status = check_citation_support(
            statement, evidence or restated_evidence, references, evidence_blocks
        )

        if grounded_evidence_status:
            supported_count += 1
            if direct_evidence_status:
                direct_supported_count += 1
            else:
                restated_supported_count += 1
        else:
            unsupported_count += 1
        if any_support_status:
            any_supported_count += 1
        if graph_derived_status:
            graph_derived_count += 1
        if unsupported_by_both_status:
            unsupported_by_both_count += 1
        cited_nums = _citation_nums(statement)
        if cited_nums and citation_status is None:
            citation_unverifiable_count += 1
        elif citation_status is not None:
            citation_checked += 1
            if citation_status:
                citation_correct_count += 1
        if graph_applicable_status:
            graph_applicable_count += 1
        if graph_applicable_status and graph_status:
            graph_aligned_count += 1
        if statement_type in {"graph_inference", "unsupported"}:
            inference_count += 1

        cited_dois = []
        for n in cited_nums:
            for block in evidence_blocks:
                if block.get("reference_index") != n:
                    continue
                doi = block.get("doi")
                if doi:
                    cited_dois.append(doi)

        results.append({
            "sentence_id": f"sent_{idx:04d}",
            "statement": statement,
            "support_status": support_status,
            "grounded_evidence_status": grounded_evidence_status,
            "any_support_status": any_support_status,
            "direct_evidence_status": direct_evidence_status,
            "restated_evidence_status": restated_status,
            "supporting_evidence": (
                evidence if direct_evidence_status else restated_evidence
            ),
            "citation_status": citation_status,
            "citation_verification_status": (
                "verified_supported" if citation_status is True
                else "verified_unsupported" if citation_status is False
                else "unverifiable_no_snippet" if cited_nums
                else "not_cited"
            ),
            "cited_references": cited_nums,
            "cited_dois": cited_dois,
            "graph_alignment": graph_status,
            "graph_alignment_applicable": graph_applicable_status,
            "graph_alignment_status": (
                "aligned"
                if graph_applicable_status and graph_status
                else "not_aligned"
                if graph_applicable_status
                else "not_applicable"
            ),
            "graph_alignment_details": graph_details,
            "graph_derived_status": graph_derived_status,
            "graph_only_status": graph_derived_status,
            "unsupported_status": unsupported_status,
            "unsupported_by_both_status": unsupported_by_both_status,
            "unsupported_by_both": unsupported_by_both_status,
            "support_basis": (
                "direct_snippet"
                if direct_evidence_status
                else "snippet_restatement"
                if restated_status
                else ("graph_only" if graph_status else "none")
            ),
            "statement_type": statement_type,
        })

    total = len(statements)
    citation_accuracy = citation_correct_count / citation_checked if citation_checked else None
    coverage = supported_count / total if total else 0.0
    direct_coverage = direct_supported_count / total if total else 0.0
    snippet_count = sum(1 for block in evidence_blocks if _evidence_text(block))
    all_inference_report = bool(
        total
        and all(
            _is_inference_statement(item["statement"])
            or bool(item["graph_derived_status"])
            for item in results
        )
    )

    manual_reasons = []
    if snippet_count == 0:
        manual_reasons.append("No snippet-level evidence was retrieved.")
    if total == 0:
        manual_reasons.append("No evaluable factual statements were found.")
    elif supported_count == 0:
        manual_reasons.append("No evaluated statement has direct snippet-level evidence.")
    elif unsupported_count:
        manual_reasons.append(
            f"{unsupported_count} of {total} evaluated statements lack direct snippet-level evidence."
        )
    if all_inference_report:
        manual_reasons.append("Every evaluated statement is marked or classified as inference.")
    failed_citations = citation_checked - citation_correct_count
    if failed_citations:
        manual_reasons.append(
            f"{failed_citations} cited statement(s) failed snippet-level citation support."
        )
    if citation_unverifiable_count:
        manual_reasons.append(
            f"{citation_unverifiable_count} cited statement(s) could not be verified "
            "because no citation-linked snippet was retrieved."
        )

    if citation_checked and citation_unverifiable_count:
        citation_verification_status = "partially_verifiable"
    elif citation_checked and citation_correct_count == citation_checked:
        citation_verification_status = "verified"
    elif citation_checked and 0 < citation_correct_count < citation_checked:
        citation_verification_status = "partially_supported"
    elif citation_checked:
        citation_verification_status = "unsupported"
    elif citation_unverifiable_count:
        citation_verification_status = "unverifiable"
    else:
        citation_verification_status = "not_applicable"

    manual_verification = bool(manual_reasons)
    manual_reason = " ".join(manual_reasons) if manual_reasons else (
        "Direct snippet evidence was found for all evaluated statements; "
        "no citation-support failure was detected."
    )

    graph_alignment = (
        round(graph_aligned_count / graph_applicable_count, 3)
        if graph_applicable_count
        else 0.0
    )
    edge_evidence_coverage, endpoint_conversion_supported = (
        _pathway_evidence_coverage(pathway_graph, evidence_blocks)
    )
    source_quality = _source_traceability_assessment(evidence_blocks, references)
    reliability = _overall_reliability_assessment(
        total=total,
        snippet_count=snippet_count,
        evidence_coverage=coverage,
        citation_accuracy=citation_accuracy,
        citation_checked=citation_checked,
        citation_unverifiable_count=citation_unverifiable_count,
        graph_alignment=graph_alignment,
        graph_alignment_applicable_count=graph_applicable_count,
        source_quality_score=source_quality["score"],
        edge_evidence_coverage=edge_evidence_coverage,
        endpoint_conversion_supported=endpoint_conversion_supported,
        unsupported_count=unsupported_count,
    )

    return {
        "total_statements": total,
        "supported_statements": supported_count,
        "grounded_statements": supported_count,
        "direct_evidence_statements": direct_supported_count,
        "restated_evidence_statements": restated_supported_count,
        "unsupported_statements": unsupported_count,
        "statements_without_direct_evidence": unsupported_count,
        "unsupported_by_both_statements": unsupported_by_both_count,
        "unsupported_by_both": unsupported_by_both_count,
        "any_supported_statements": any_supported_count,
        "evidence_coverage": round(coverage, 3),
        "statement_evidence_coverage": round(coverage, 3),
        "direct_evidence_coverage": round(direct_coverage, 3),
        "any_support_coverage": round(any_supported_count / total, 3) if total else 0.0,
        "citation_accuracy": round(citation_accuracy, 3) if citation_accuracy is not None else None,
        "citation_checked_statements": citation_checked,
        "citation_correct_statements": citation_correct_count,
        "citation_unverifiable_statements": citation_unverifiable_count,
        "citation_unverifiable_count": citation_unverifiable_count,
        "citation_verification_status": citation_verification_status,
        "graph_alignment": graph_alignment,
        "graph_aligned_statements": graph_aligned_count,
        "graph_alignment_applicable_statements": graph_applicable_count,
        "graph_alignment_status": (
            "evaluated" if graph_applicable_count else "not_applicable"
        ),
        "graph_derived_statements": graph_derived_count,
        "graph_only_statements": graph_derived_count,
        "inference_statements": inference_count,
        "inference_only_statements": inference_count,
        "all_inference_report": all_inference_report,
        "retrieved_snippet_count": snippet_count,
        "edge_evidence_coverage": edge_evidence_coverage,
        "endpoint_conversion_supported": endpoint_conversion_supported,
        "source_quality_score": source_quality["score"],
        "source_quality_level": source_quality["level"],
        "source_quality_details": source_quality["details"],
        "source_quality_basis": source_quality["basis"],
        "overall_reliability_score": reliability["score"],
        "overall_reliability_level": reliability["level"],
        "overall_reliability_summary": reliability["summary"],
        "overall_reliability_reasons": reliability["reasons"],
        "manual_verification": manual_verification,
        "manual_verification_required": manual_verification,
        "manual_verification_status": "required" if manual_verification else "not_required",
        "manual_verification_reason": manual_reason,
        "sentence_level_results": results,
    }