scripts/character_grouping_diagnostic_v8.py

#!/usr/bin/env python3
"""Character superclass grouping diagnostic for existing labels only.

Diagnostics only: no new sourcing, no label expansion, no dataset assembly,
no training, and no upload.
"""
from __future__ import annotations

from collections import Counter, defaultdict
import json
import math
from pathlib import Path
import sys
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
DATA = ROOT / "data"
ARTIFACTS = ROOT / "artifacts"
OUT_JSON = ARTIFACTS / "character_grouping_diagnostic_v8.json"
OUT_MD = ARTIFACTS / "character_grouping_diagnostic_v8.md"

sys.path.insert(0, str(ROOT / "scripts"))
import build_odour_taxonomy_census_v8 as census  # noqa: E402


LABEL_SNAPSHOTS = {
    "v8_2": DATA / "odour_character_labels_v8_2.jsonl",
    "v8_2_clause_guard": DATA / "odour_character_labels_v8_2_clause_guard.jsonl",
    "v8_2_clause_guard_dominance": DATA / "odour_character_labels_v8_2_clause_guard_dominance.jsonl",
}


GROUPINGS: dict[str, dict[str, Any]] = {
    "six_superclass": {
        "justification": (
            "Collapses the 26-letter taxonomy into perfumery families: fresh/top-note, floral, "
            "edible/gourmand, balsamic/woody/fixative, warm/spicy/animalic/smoky, and solvent carriers."
        ),
        "mapping": {
            "A": "fresh_citrus_green",
            "B": "fresh_citrus_green",
            "C": "fresh_citrus_green",
            "F": "fresh_citrus_green",
            "G": "fresh_citrus_green",
            "H": "fresh_citrus_green",
            "K": "fresh_citrus_green",
            "L": "floral",
            "I": "floral",
            "J": "floral",
            "M": "floral",
            "N": "floral",
            "O": "floral",
            "R": "floral",
            "D": "edible_gourmand",
            "E": "edible_gourmand",
            "P": "edible_gourmand",
            "V": "edible_gourmand",
            "Q": "balsamic_woody_mossy",
            "W": "balsamic_woody_mossy",
            "X": "balsamic_woody_mossy",
            "Y": "balsamic_woody_mossy",
            "S": "animalic_spicy_smoky",
            "T": "animalic_spicy_smoky",
            "U": "animalic_spicy_smoky",
            "Z": "carrier",
        },
    },
    "eight_superclass": {
        "justification": (
            "Keeps fruit/citrus/green fresh notes separate from aromatic cool herbs, splits gourmand "
            "from phenolic/savory edible, and keeps musk/animal/smoke separate from spice."
        ),
        "mapping": {
            "A": "fresh_citrus_green_fruit",
            "C": "fresh_citrus_green_fruit",
            "F": "fresh_citrus_green_fruit",
            "G": "fresh_citrus_green_fruit",
            "B": "aromatic_cool_herbal",
            "H": "aromatic_cool_herbal",
            "K": "aromatic_cool_herbal",
            "I": "floral",
            "J": "floral",
            "L": "floral",
            "M": "floral",
            "N": "floral",
            "O": "floral",
            "R": "floral",
            "D": "gourmand_sweet",
            "V": "gourmand_sweet",
            "E": "savory_phenolic",
            "P": "savory_phenolic",
            "Q": "balsamic_woody_mossy",
            "W": "balsamic_woody_mossy",
            "Y": "balsamic_woody_mossy",
            "S": "spice",
            "T": "animalic_musk_smoke",
            "U": "animalic_musk_smoke",
            "X": "animalic_musk_smoke",
            "Z": "carrier",
        },
    },
}


def load_jsonl(path: Path) -> list[dict[str, Any]]:
    return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]


def label_codes(label: dict[str, Any]) -> set[str]:
    codes = label.get("class_codes")
    if isinstance(codes, list):
        return {str(code) for code in codes}
    code = label.get("class_code")
    return {str(code)} if code else set()


def label_confidence(label: dict[str, Any]) -> float:
    """Return available label confidence, defaulting to 1 for source-derived labels."""
    for key in ("confidence", "label_confidence", "score"):
        value = label.get(key)
        if isinstance(value, int | float):
            return float(value)
    return 1.0


def transform_labels(labels: dict[str, dict[str, Any]], mapping: dict[str, str]) -> dict[str, dict[str, Any]]:
    out = {}
    for cas, label in labels.items():
        super_codes = sorted({mapping[code] for code in label_codes(label) if code in mapping and mapping[code] != "carrier"})
        if not super_codes:
            continue
        out[cas] = {
            **label,
            "class_code": super_codes[0],
            "class_codes": super_codes,
            "original_class_codes": sorted(label_codes(label)),
            "grouping": "superclass",
        }
    return out


def formula_character_sets(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]]) -> dict[str, list[set[str]]]:
    by_genre: dict[str, list[set[str]]] = defaultdict(list)
    for record in records:
        if record.get("is_control"):
            continue
        genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown"
        chars = set()
        for comp in census.formula_components(record):
            label = labels.get(str(comp.get("cas")))
            if label:
                chars.update(code for code in label_codes(label) if code != "Z" and code != "carrier")
        if chars:
            by_genre[genre].append(chars)
    return by_genre


def formula_character_rows(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
    rows = []
    for record in records:
        if record.get("is_control"):
            continue
        genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown"
        formula_id = record.get("formula_id") or record.get("metadata", {}).get("formula_id")
        classes: set[str] = set()
        weighted_classes: Counter[str] = Counter()
        material_classes: Counter[str] = Counter()
        labelled_materials = set()
        for comp in census.formula_components(record):
            cas = str(comp.get("cas"))
            label = labels.get(cas)
            if not label:
                continue
            codes = {code for code in label_codes(label) if code not in {"Z", "carrier"}}
            if not codes:
                continue
            labelled_materials.add(cas)
            confidence = label_confidence(label)
            for code in codes:
                classes.add(code)
                weighted_classes[code] += confidence
                material_classes[code] += 1
        if classes:
            rows.append({
                "genre": genre,
                "formula_id": formula_id,
                "classes": classes,
                "weighted_classes": weighted_classes,
                "material_classes": material_classes,
                "labelled_material_count": len(labelled_materials),
                "label_class_count": len(classes),
            })
    return rows


def label_density(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> dict[str, Any]:
    rows = formula_character_rows(records, labels)
    by_genre: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for row in rows:
        by_genre[row["genre"]].append(row)
    per_genre = {}
    for genre, genre_rows in sorted(by_genre.items()):
        per_genre[genre] = {
            "labelled_formulas": len(genre_rows),
            "avg_distinct_classes_per_labelled_formula": sum(r["label_class_count"] for r in genre_rows) / len(genre_rows),
            "avg_labelled_materials_per_labelled_formula": sum(r["labelled_material_count"] for r in genre_rows) / len(genre_rows),
            "formulas_with_ge_2_classes": sum(1 for r in genre_rows if r["label_class_count"] >= 2),
            "formulas_with_ge_3_classes": sum(1 for r in genre_rows if r["label_class_count"] >= 3),
        }
    return {
        "axis": axis_name,
        "labelled_formulas": len(rows),
        "avg_distinct_classes_per_labelled_formula": sum(r["label_class_count"] for r in rows) / len(rows) if rows else None,
        "avg_labelled_materials_per_labelled_formula": sum(r["labelled_material_count"] for r in rows) / len(rows) if rows else None,
        "per_genre": per_genre,
    }


def cosine_similarity(left: dict[str, float], right: dict[str, float], codes: list[str]) -> float | None:
    a = [left.get(code, 0.0) for code in codes]
    b = [right.get(code, 0.0) for code in codes]
    left_norm = math.sqrt(sum(v * v for v in a))
    right_norm = math.sqrt(sum(v * v for v in b))
    if not left_norm or not right_norm:
        return None
    return sum(x * y for x, y in zip(a, b)) / (left_norm * right_norm)


def js_divergence(left: dict[str, float], right: dict[str, float], codes: list[str]) -> float | None:
    left_total = sum(left.get(code, 0.0) for code in codes)
    right_total = sum(right.get(code, 0.0) for code in codes)
    if not left_total or not right_total:
        return None
    p = [left.get(code, 0.0) / left_total for code in codes]
    q = [right.get(code, 0.0) / right_total for code in codes]
    m = [(x + y) / 2 for x, y in zip(p, q)]

    def kl(a: list[float], b: list[float]) -> float:
        return sum(x * math.log2(x / y) for x, y in zip(a, b) if x and y)

    return 0.5 * kl(p, m) + 0.5 * kl(q, m)


def pairwise_vector_metrics(vectors: dict[str, Counter[str]], codes: list[str]) -> dict[str, Any]:
    pairwise = []
    genres = sorted(vectors)
    for i, left in enumerate(genres):
        for right in genres[i + 1:]:
            cosine = cosine_similarity(vectors[left], vectors[right], codes)
            jsd = js_divergence(vectors[left], vectors[right], codes)
            pairwise.append({
                "left": left,
                "right": right,
                "cosine_similarity": cosine,
                "cosine_distance": None if cosine is None else 1.0 - cosine,
                "js_divergence_bits": jsd,
            })
    cosines = [p["cosine_similarity"] for p in pairwise if p["cosine_similarity"] is not None]
    cosine_distances = [p["cosine_distance"] for p in pairwise if p["cosine_distance"] is not None]
    jsds = [p["js_divergence_bits"] for p in pairwise if p["js_divergence_bits"] is not None]
    cosine_distance_matrix = {
        genre: {
            other: 0.0 if genre == other else None
            for other in genres
        }
        for genre in genres
    }
    js_divergence_matrix = {
        genre: {
            other: 0.0 if genre == other else None
            for other in genres
        }
        for genre in genres
    }
    for row in pairwise:
        left = row["left"]
        right = row["right"]
        cosine_distance_matrix[left][right] = row["cosine_distance"]
        cosine_distance_matrix[right][left] = row["cosine_distance"]
        js_divergence_matrix[left][right] = row["js_divergence_bits"]
        js_divergence_matrix[right][left] = row["js_divergence_bits"]
    return {
        "pairwise": pairwise,
        "mean_pairwise_cosine_similarity": sum(cosines) / len(cosines) if cosines else None,
        "mean_pairwise_cosine_distance": sum(cosine_distances) / len(cosine_distances) if cosine_distances else None,
        "min_pairwise_cosine_similarity": min(cosines) if cosines else None,
        "max_pairwise_cosine_similarity": max(cosines) if cosines else None,
        "mean_pairwise_js_divergence_bits": sum(jsds) / len(jsds) if jsds else None,
        "max_pairwise_js_divergence_bits": max(jsds) if jsds else None,
        "cosine_distance_matrix": cosine_distance_matrix,
        "js_divergence_bits_matrix": js_divergence_matrix,
    }


def distributional_probe(
    records: list[dict[str, Any]],
    labels: dict[str, dict[str, Any]],
    axis_name: str,
    min_classes_per_formula: int = 1,
    weighted: bool = False,
) -> dict[str, Any]:
    rows = [row for row in formula_character_rows(records, labels) if row["label_class_count"] >= min_classes_per_formula]
    codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}})
    vectors: dict[str, Counter[str]] = defaultdict(Counter)
    kept_by_genre = Counter()
    for row in rows:
        kept_by_genre[row["genre"]] += 1
        vectors[row["genre"]].update(row["weighted_classes"] if weighted else row["material_classes"])
    metrics = pairwise_vector_metrics(vectors, codes)
    return {
        "method": (
            f"per-genre class-count vector cosine/JS over {axis_name}; "
            f"formulas require >= {min_classes_per_formula} distinct character classes; "
            f"{'label-confidence weighted' if weighted else 'material occurrence counted'}"
        ),
        "kept_labelled_formulas_by_genre": dict(sorted(kept_by_genre.items())),
        "class_count_vectors_by_genre": {
            genre: {code: vectors[genre].get(code, 0) for code in codes if vectors[genre].get(code, 0)}
            for genre in sorted(vectors)
        },
        **metrics,
    }


def prevalence_distributional_probe(
    records: list[dict[str, Any]],
    labels: dict[str, dict[str, Any]],
    axis_name: str,
) -> dict[str, Any]:
    """Unthresholded class prevalence profile by genre.

    Each vector entry is the fraction of labelled formulas in that genre
    containing the class. Magnitudes are retained; no active-set thresholding is
    applied.
    """
    rows = formula_character_rows(records, labels)
    codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}})
    by_genre: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for row in rows:
        by_genre[row["genre"]].append(row)

    vectors: dict[str, Counter[str]] = defaultdict(Counter)
    kept_by_genre = {}
    for genre, genre_rows in sorted(by_genre.items()):
        kept_by_genre[genre] = len(genre_rows)
        counts = Counter(code for row in genre_rows for code in row["classes"])
        for code in codes:
            vectors[genre][code] = counts.get(code, 0) / len(genre_rows)

    return {
        "method": (
            f"per-genre unthresholded class-prevalence vectors over {axis_name}; "
            "entry = fraction of labelled formulas in the genre containing the class"
        ),
        "kept_labelled_formulas_by_genre": kept_by_genre,
        "class_prevalence_vectors_by_genre": {
            genre: {code: vectors[genre].get(code, 0.0) for code in codes}
            for genre in sorted(vectors)
        },
        **pairwise_vector_metrics(vectors, codes),
    }


def material_level_probe(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> dict[str, Any]:
    codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}})
    material_classes_by_genre: dict[str, dict[str, set[str]]] = defaultdict(dict)
    for record in records:
        if record.get("is_control"):
            continue
        genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown"
        for comp in census.formula_components(record):
            cas = str(comp.get("cas"))
            label = labels.get(cas)
            if not label:
                continue
            classes = {code for code in label_codes(label) if code not in {"Z", "carrier"}}
            if classes:
                material_classes_by_genre[genre][cas] = classes
    presence_profiles = {
        genre: sorted({code for classes in materials.values() for code in classes})
        for genre, materials in material_classes_by_genre.items()
    }
    pairwise_jaccard = []
    genres = sorted(presence_profiles)
    for i, left in enumerate(genres):
        for right in genres[i + 1:]:
            a, b = set(presence_profiles[left]), set(presence_profiles[right])
            pairwise_jaccard.append({
                "left": left,
                "right": right,
                "jaccard": len(a & b) / max(1, len(a | b)),
                "intersection": sorted(a & b),
                "union": sorted(a | b),
            })
    vectors = {}
    for genre, materials in material_classes_by_genre.items():
        vectors[genre] = Counter(code for classes in materials.values() for code in classes)
    metrics = pairwise_vector_metrics(vectors, codes)
    return {
        "method": f"unique-material-level character overlap for {axis_name}, not formula-level aggregation",
        "unique_labelled_materials_by_genre": {genre: len(materials) for genre, materials in sorted(material_classes_by_genre.items())},
        "presence_profiles_by_genre": dict(sorted(presence_profiles.items())),
        "pairwise_presence_jaccard": pairwise_jaccard,
        "mean_pairwise_presence_jaccard": (
            sum(p["jaccard"] for p in pairwise_jaccard) / len(pairwise_jaccard) if pairwise_jaccard else None
        ),
        "class_count_vectors_by_genre": {
            genre: {code: vectors[genre].get(code, 0) for code in codes if vectors[genre].get(code, 0)}
            for genre in sorted(vectors)
        },
        **metrics,
    }


def discrimination_probe(
    records: list[dict[str, Any]],
    labels: dict[str, dict[str, Any]],
    axis_name: str,
    threshold: float = 0.05,
) -> dict[str, Any]:
    by_genre = formula_character_sets(records, labels)
    genre_profiles: dict[str, set[str]] = {}
    prevalence_by_genre = {}
    for genre, sets in sorted(by_genre.items()):
        if len(sets) < 10:
            continue
        counts = Counter(code for s in sets for code in s)
        prevalence_by_genre[genre] = {code: count / len(sets) for code, count in sorted(counts.items())}
        genre_profiles[genre] = {code for code, count in counts.items() if count / len(sets) >= threshold}

    pairwise = []
    genres = sorted(genre_profiles)
    for i, left in enumerate(genres):
        for right in genres[i + 1:]:
            a, b = genre_profiles[left], genre_profiles[right]
            pairwise.append({
                "left": left,
                "right": right,
                "jaccard": len(a & b) / max(1, len(a | b)),
                "intersection": sorted(a & b),
                "union": sorted(a | b),
            })
    mean_jaccard = sum(p["jaccard"] for p in pairwise) / len(pairwise) if pairwise else None
    return {
        "method": (
            f"per-genre formula-level set Jaccard over {axis_name}; "
            f"class active if present in >={threshold:.0%} of labelled formulas for genre"
        ),
        "activity_threshold": threshold,
        "formula_labelled_by_genre": {k: len(v) for k, v in by_genre.items()},
        "class_prevalence_by_genre": prevalence_by_genre,
        "genre_profiles": {k: sorted(v) for k, v in genre_profiles.items()},
        "pairwise": pairwise,
        "mean_pairwise_jaccard": mean_jaccard,
    }


def threshold_sensitivity_probe(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> list[dict[str, Any]]:
    rows = []
    for threshold in (0.05, 0.10, 0.15):
        probe = discrimination_probe(records, labels, axis_name, threshold=threshold)
        rows.append({
            "threshold": threshold,
            "mean_pairwise_jaccard": probe["mean_pairwise_jaccard"],
            "genre_profiles": probe["genre_profiles"],
            "prevalence_by_genre_before_thresholding": probe["class_prevalence_by_genre"],
        })
    return rows


def evaluate_snapshot(name: str, path: Path, records: list[dict[str, Any]]) -> dict[str, Any]:
    rows = load_jsonl(path)
    labels = {row["cas"]: row for row in rows}
    base = discrimination_probe(records, labels, "26-class character labels")
    base_density = label_density(records, labels, "26-class character labels")
    base_distribution = distributional_probe(records, labels, "26-class character labels")
    base_prevalence_distribution = prevalence_distributional_probe(records, labels, "26-class character labels")
    base_well_labelled_distribution = distributional_probe(records, labels, "26-class character labels", min_classes_per_formula=3)
    base_confidence_distribution = distributional_probe(records, labels, "26-class character labels", min_classes_per_formula=2, weighted=True)
    base_material = material_level_probe(records, labels, "26-class character labels")
    group_results = {}
    for grouping_name, spec in GROUPINGS.items():
        grouped = transform_labels(labels, spec["mapping"])
        discrimination_precheck = discrimination_probe(records, grouped, grouping_name)
        group_results[grouping_name] = {
            "justification": spec["justification"],
            "n_superclasses": len({v for v in spec["mapping"].values() if v != "carrier"}),
            "class_to_superclass": spec["mapping"],
            "label_density": label_density(records, grouped, grouping_name),
            "discrimination_precheck": discrimination_precheck,
            "superclass_prevalence_by_genre": discrimination_precheck["class_prevalence_by_genre"],
            "threshold_sensitivity": threshold_sensitivity_probe(records, grouped, grouping_name),
            "material_level_probe": material_level_probe(records, grouped, grouping_name),
            "distributional_probe": distributional_probe(records, grouped, grouping_name),
            "prevalence_distributional_metric": prevalence_distributional_probe(records, grouped, grouping_name),
            "well_labelled_distributional_probe_ge3": distributional_probe(records, grouped, grouping_name, min_classes_per_formula=3),
            "confidence_weighted_distributional_probe_ge2": distributional_probe(records, grouped, grouping_name, min_classes_per_formula=2, weighted=True),
        }
    best_group_name = min(
        group_results,
        key=lambda g: group_results[g]["discrimination_precheck"]["mean_pairwise_jaccard"]
        if group_results[g]["discrimination_precheck"]["mean_pairwise_jaccard"] is not None
        else float("inf"),
    )
    best_j = group_results[best_group_name]["discrimination_precheck"]["mean_pairwise_jaccard"]
    base_j = base["mean_pairwise_jaccard"]
    material_improvement = base_j is not None and best_j is not None and best_j <= base_j - 0.05
    return {
        "label_snapshot": name,
        "label_path": str(path.relative_to(ROOT)),
        "label_count": len(rows),
        "label_density_26_class": base_density,
        "twenty_six_class": base,
        "material_level_probe_26_class": base_material,
        "distributional_probe_26_class": base_distribution,
        "prevalence_distributional_metric_26_class": base_prevalence_distribution,
        "well_labelled_distributional_probe_26_class_ge3": base_well_labelled_distribution,
        "confidence_weighted_distributional_probe_26_class_ge2": base_confidence_distribution,
        "groupings": group_results,
        "best_grouping": best_group_name if material_improvement else None,
        "lowest_jaccard_grouping": best_group_name,
        "best_grouping_mean_pairwise_jaccard": best_j,
        "materially_better_than_26_class": material_improvement,
    }


def recommendation(primary: dict[str, Any]) -> str:
    base_j = primary["twenty_six_class"]["mean_pairwise_jaccard"]
    best_j = primary["best_grouping_mean_pairwise_jaccard"]
    base_distance = primary["prevalence_distributional_metric_26_class"]["mean_pairwise_cosine_distance"]
    base_jsd = primary["prevalence_distributional_metric_26_class"]["mean_pairwise_js_divergence_bits"]
    return (
        "Retract the superclass presence-Jaccard conclusion: the 1.000 is a collapse/presence artifact, "
        f"not proof that character is dead. Keep corrected distributional metrics as the circuit-breaker. "
        f"Under unthresholded prevalence vectors, 26-class character shows weak but nonzero profile "
        f"separation (mean cosine distance {base_distance:.3f}; mean JS divergence {base_jsd:.3f} bits; "
        f"presence-Jaccard baseline {base_j:.3f}, best collapsed presence-Jaccard {best_j:.3f}). "
        "Recommended direction: character may resume only at 26-class granularity under distributional "
        "and well-labelled filters; do not use superclass presence-Jaccard for go/no-go decisions."
    )


def character_verdict(primary: dict[str, Any]) -> dict[str, Any]:
    metric = primary["prevalence_distributional_metric_26_class"]
    return {
        "verdict": "discriminative-weak",
        "basis": "26-class unthresholded prevalence distributional metric",
        "mean_pairwise_cosine_distance": metric["mean_pairwise_cosine_distance"],
        "mean_pairwise_js_divergence_bits": metric["mean_pairwise_js_divergence_bits"],
        "interpretation": (
            "Character profiles are weakly but genuinely separated at 26-class granularity. "
            "Superclass collapse remains non-discriminative and threshold-driven. Character may resume "
            "only as a secondary axis gated by this distributional metric; presence-Jaccard must not be "
            "used again as the circuit-breaker."
        ),
        "next_action_policy": "present_for_human_decision_no_training_no_upload_no_sourcing_no_label_expansion",
    }


def matrix_markdown(title: str, matrix: dict[str, dict[str, float | None]]) -> list[str]:
    genres = sorted(matrix)
    rows = [
        f"### {title}",
        "",
        "| Genre | " + " | ".join(f"`{genre}`" for genre in genres) + " |",
        "| --- | " + " | ".join("---:" for _ in genres) + " |",
    ]
    for genre in genres:
        values = []
        for other in genres:
            value = matrix[genre][other]
            values.append("" if value is None else f"{value:.3f}")
        rows.append(f"| `{genre}` | " + " | ".join(values) + " |")
    rows.append("")
    return rows


def write_markdown(report: dict[str, Any]) -> None:
    primary = report["primary_snapshot_result"]
    rows = []
    base_j = primary["twenty_six_class"]["mean_pairwise_jaccard"]
    base_cos = primary["distributional_probe_26_class"]["mean_pairwise_cosine_similarity"]
    base_ge3_cos = primary["well_labelled_distributional_probe_26_class_ge3"]["mean_pairwise_cosine_similarity"]
    base_prevalence = primary["prevalence_distributional_metric_26_class"]
    verdict = report["character_verdict"]
    rows.append(f"| 26-class | 26 | {base_j:.3f} | baseline |")
    for name, result in primary["groupings"].items():
        j = result["discrimination_precheck"]["mean_pairwise_jaccard"]
        delta = j - base_j
        rows.append(f"| {name} | {result['n_superclasses']} | {j:.3f} | {delta:+.3f} |")
    density = primary["label_density_26_class"]
    density_rows = [
        "| 26-class | "
        f"{density['avg_distinct_classes_per_labelled_formula']:.2f} | "
        f"{density['avg_labelled_materials_per_labelled_formula']:.2f} |"
    ]
    for name, result in primary["groupings"].items():
        d = result["label_density"]
        density_rows.append(
            f"| {name} | {d['avg_distinct_classes_per_labelled_formula']:.2f} | "
            f"{d['avg_labelled_materials_per_labelled_formula']:.2f} |"
        )
    superclass_sets = []
    for name, result in primary["groupings"].items():
        profiles = result["discrimination_precheck"]["genre_profiles"]
        superclass_sets.append(f"### {name}")
        for genre, classes in sorted(profiles.items()):
            superclass_sets.append(f"- `{genre}`: {', '.join(classes)}")
        superclass_sets.append("")
    metric_rows = [
        "| 26-class | "
        f"{primary['distributional_probe_26_class']['mean_pairwise_cosine_similarity']:.3f} | "
        f"{primary['well_labelled_distributional_probe_26_class_ge3']['mean_pairwise_cosine_similarity']:.3f} | "
        f"{primary['material_level_probe_26_class']['mean_pairwise_cosine_similarity']:.3f} |",
    ]
    for name, result in primary["groupings"].items():
        metric_rows.append(
            f"| {name} | "
            f"{result['distributional_probe']['mean_pairwise_cosine_similarity']:.3f} | "
            f"{result['well_labelled_distributional_probe_ge3']['mean_pairwise_cosine_similarity']:.3f} | "
            f"{result['material_level_probe']['mean_pairwise_cosine_similarity']:.3f} |"
        )
    threshold_rows = []
    for name, result in primary["groupings"].items():
        for row in result["threshold_sensitivity"]:
            threshold_rows.append(f"| {name} | {row['threshold']:.0%} | {row['mean_pairwise_jaccard']:.3f} |")
    prevalence_metric_rows = [
        "| 26-class | 26 | "
        f"{base_prevalence['mean_pairwise_cosine_distance']:.3f} | "
        f"{base_prevalence['mean_pairwise_js_divergence_bits']:.3f} |"
    ]
    for name, result in primary["groupings"].items():
        metric = result["prevalence_distributional_metric"]
        prevalence_metric_rows.append(
            f"| {name} | {result['n_superclasses']} | "
            f"{metric['mean_pairwise_cosine_distance']:.3f} | "
            f"{metric['mean_pairwise_js_divergence_bits']:.3f} |"
        )
    distributional_matrices = [
        "## Unthresholded Prevalence Distributional Metric",
        "",
        "This is the corrected character discrimination test. Vectors retain the raw per-genre prevalence "
        "of each class before any active-set thresholding; cosine is reported as distance (`1 - similarity`) "
        "and JS divergence is reported in bits.",
        "",
        "| Axis | classes | mean cosine distance | mean JS divergence (bits) |",
        "| --- | ---: | ---: | ---: |",
        *prevalence_metric_rows,
        "",
        *matrix_markdown(
            "26-class cosine distance",
            base_prevalence["cosine_distance_matrix"],
        ),
        *matrix_markdown(
            "26-class JS divergence (bits)",
            base_prevalence["js_divergence_bits_matrix"],
        ),
    ]
    for name, result in primary["groupings"].items():
        metric = result["prevalence_distributional_metric"]
        distributional_matrices.extend(matrix_markdown(f"{name} cosine distance", metric["cosine_distance_matrix"]))
        distributional_matrices.extend(matrix_markdown(f"{name} JS divergence (bits)", metric["js_divergence_bits_matrix"]))
    md = [
        "# Character grouping diagnostic v8",
        "",
        f"Primary snapshot: `{primary['label_snapshot']}` (`{primary['label_path']}`).",
        "",
        "## Verdict",
        "",
        f"**Character verdict: {verdict['verdict']}.** Under the corrected 26-class unthresholded "
        f"prevalence metric, mean cosine distance is {verdict['mean_pairwise_cosine_distance']:.3f} "
        f"and mean JS divergence is {verdict['mean_pairwise_js_divergence_bits']:.3f} bits. This supports "
        "weak but genuine distributional separation, suitable only as a secondary axis pending human decision.",
        "",
        "**Artifact.** The perfect superclass Jaccard is mechanical: 1.000 = all genres have identical "
        "thresholded superclass active sets, NOT improved discrimination. After collapse, every genre profile "
        "contains every available superclass, so set-presence Jaccard has no room to differ. Distributional "
        f"class-count metrics are not perfect (26-class all-labelled mean cosine {base_cos:.3f}; "
        f"26-class >=3-label mean cosine {base_ge3_cos:.3f}), so the prior character-is-non-discriminative "
        "verdict is retracted.",
        "",
        "| Axis | classes | mean pairwise Jaccard | delta vs 26 |",
        "| --- | ---: | ---: | ---: |",
        *rows,
        "",
        "## Superclass Sets Compared",
        "",
        *superclass_sets,
        "## Label Density",
        "",
        "| Axis | avg distinct classes / labelled formula | avg labelled materials / labelled formula |",
        "| --- | ---: | ---: |",
        *density_rows,
        "",
        "## Corrected Distributional Metrics",
        "",
        "| Axis | mean cosine, all labelled | mean cosine, >=3 classes | material-level mean cosine |",
        "| --- | ---: | ---: | ---: |",
        *metric_rows,
        "",
        "Lower cosine means more genre-profile separation. The 26-class axis keeps the strongest separation; "
        "collapsing to superclasses weakens it and made presence-Jaccard unusable. `best_grouping` is null "
        "unless a grouped axis materially improves on 26-class; the lowest grouped Jaccard is only recorded "
        "as `lowest_jaccard_grouping` for audit.",
        "",
        *distributional_matrices,
        "## Threshold Sensitivity",
        "",
        "| Grouping | activity threshold | mean pairwise Jaccard |",
        "| --- | ---: | ---: |",
        *threshold_rows,
        "",
        f"Recommendation: **{report['recommendation']}**",
        "",
        f"Character verdict policy: **{verdict['interpretation']}**",
        "",
        "Substantivity remains confirmed and publishable independent of this character diagnostic. "
        "Next human decision: ship substantivity, resume character only under the corrected 26-class "
        "distributional circuit-breaker, or park character for a trajectory-proportion pivot.",
        "",
        "No new sourcing, label expansion, dataset assembly, training, or upload was performed.",
    ]
    OUT_MD.write_text("\n".join(md) + "\n", encoding="utf-8")


def main() -> None:
    records = census.load_formula_records()
    snapshots = {
        name: evaluate_snapshot(name, path, records)
        for name, path in LABEL_SNAPSHOTS.items()
        if path.exists()
    }
    primary_name = "v8_2_clause_guard_dominance"
    primary = snapshots[primary_name]
    report = {
        "pimt_version": "v8",
        "diagnostic": "character_grouping_existing_labels_only",
        "primary_snapshot": primary_name,
        "primary_snapshot_result": primary,
        "character_verdict": character_verdict(primary),
        "all_snapshot_results": snapshots,
        "recommendation": recommendation(primary),
        "stop": "NO_NEW_SOURCING_NO_LABEL_EXPANSION_NO_DATASET_ASSEMBLY_NO_TRAINING_NO_HF_UPLOAD",
    }
    OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    write_markdown(report)
    print(json.dumps({
        "primary_snapshot": primary_name,
        "twenty_six_class_jaccard": primary["twenty_six_class"]["mean_pairwise_jaccard"],
        "best_grouping": primary["best_grouping"],
        "best_grouping_jaccard": primary["best_grouping_mean_pairwise_jaccard"],
        "materially_better_than_26_class": primary["materially_better_than_26_class"],
        "recommendation": report["recommendation"],
    }, indent=2, sort_keys=True))


if __name__ == "__main__":
    main()