#!/usr/bin/env python3 """Character superclass grouping diagnostic for existing labels only. Diagnostics only: no new sourcing, no label expansion, no dataset assembly, no training, and no upload. """ from __future__ import annotations from collections import Counter, defaultdict import json import math from pathlib import Path import sys from typing import Any ROOT = Path(__file__).resolve().parents[1] DATA = ROOT / "data" ARTIFACTS = ROOT / "artifacts" OUT_JSON = ARTIFACTS / "character_grouping_diagnostic_v8.json" OUT_MD = ARTIFACTS / "character_grouping_diagnostic_v8.md" sys.path.insert(0, str(ROOT / "scripts")) import build_odour_taxonomy_census_v8 as census # noqa: E402 LABEL_SNAPSHOTS = { "v8_2": DATA / "odour_character_labels_v8_2.jsonl", "v8_2_clause_guard": DATA / "odour_character_labels_v8_2_clause_guard.jsonl", "v8_2_clause_guard_dominance": DATA / "odour_character_labels_v8_2_clause_guard_dominance.jsonl", } GROUPINGS: dict[str, dict[str, Any]] = { "six_superclass": { "justification": ( "Collapses the 26-letter taxonomy into perfumery families: fresh/top-note, floral, " "edible/gourmand, balsamic/woody/fixative, warm/spicy/animalic/smoky, and solvent carriers." ), "mapping": { "A": "fresh_citrus_green", "B": "fresh_citrus_green", "C": "fresh_citrus_green", "F": "fresh_citrus_green", "G": "fresh_citrus_green", "H": "fresh_citrus_green", "K": "fresh_citrus_green", "L": "floral", "I": "floral", "J": "floral", "M": "floral", "N": "floral", "O": "floral", "R": "floral", "D": "edible_gourmand", "E": "edible_gourmand", "P": "edible_gourmand", "V": "edible_gourmand", "Q": "balsamic_woody_mossy", "W": "balsamic_woody_mossy", "X": "balsamic_woody_mossy", "Y": "balsamic_woody_mossy", "S": "animalic_spicy_smoky", "T": "animalic_spicy_smoky", "U": "animalic_spicy_smoky", "Z": "carrier", }, }, "eight_superclass": { "justification": ( "Keeps fruit/citrus/green fresh notes separate from aromatic cool herbs, splits gourmand " "from phenolic/savory edible, and keeps musk/animal/smoke separate from spice." ), "mapping": { "A": "fresh_citrus_green_fruit", "C": "fresh_citrus_green_fruit", "F": "fresh_citrus_green_fruit", "G": "fresh_citrus_green_fruit", "B": "aromatic_cool_herbal", "H": "aromatic_cool_herbal", "K": "aromatic_cool_herbal", "I": "floral", "J": "floral", "L": "floral", "M": "floral", "N": "floral", "O": "floral", "R": "floral", "D": "gourmand_sweet", "V": "gourmand_sweet", "E": "savory_phenolic", "P": "savory_phenolic", "Q": "balsamic_woody_mossy", "W": "balsamic_woody_mossy", "Y": "balsamic_woody_mossy", "S": "spice", "T": "animalic_musk_smoke", "U": "animalic_musk_smoke", "X": "animalic_musk_smoke", "Z": "carrier", }, }, } def load_jsonl(path: Path) -> list[dict[str, Any]]: return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] def label_codes(label: dict[str, Any]) -> set[str]: codes = label.get("class_codes") if isinstance(codes, list): return {str(code) for code in codes} code = label.get("class_code") return {str(code)} if code else set() def label_confidence(label: dict[str, Any]) -> float: """Return available label confidence, defaulting to 1 for source-derived labels.""" for key in ("confidence", "label_confidence", "score"): value = label.get(key) if isinstance(value, int | float): return float(value) return 1.0 def transform_labels(labels: dict[str, dict[str, Any]], mapping: dict[str, str]) -> dict[str, dict[str, Any]]: out = {} for cas, label in labels.items(): super_codes = sorted({mapping[code] for code in label_codes(label) if code in mapping and mapping[code] != "carrier"}) if not super_codes: continue out[cas] = { **label, "class_code": super_codes[0], "class_codes": super_codes, "original_class_codes": sorted(label_codes(label)), "grouping": "superclass", } return out def formula_character_sets(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]]) -> dict[str, list[set[str]]]: by_genre: dict[str, list[set[str]]] = defaultdict(list) for record in records: if record.get("is_control"): continue genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown" chars = set() for comp in census.formula_components(record): label = labels.get(str(comp.get("cas"))) if label: chars.update(code for code in label_codes(label) if code != "Z" and code != "carrier") if chars: by_genre[genre].append(chars) return by_genre def formula_character_rows(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: rows = [] for record in records: if record.get("is_control"): continue genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown" formula_id = record.get("formula_id") or record.get("metadata", {}).get("formula_id") classes: set[str] = set() weighted_classes: Counter[str] = Counter() material_classes: Counter[str] = Counter() labelled_materials = set() for comp in census.formula_components(record): cas = str(comp.get("cas")) label = labels.get(cas) if not label: continue codes = {code for code in label_codes(label) if code not in {"Z", "carrier"}} if not codes: continue labelled_materials.add(cas) confidence = label_confidence(label) for code in codes: classes.add(code) weighted_classes[code] += confidence material_classes[code] += 1 if classes: rows.append({ "genre": genre, "formula_id": formula_id, "classes": classes, "weighted_classes": weighted_classes, "material_classes": material_classes, "labelled_material_count": len(labelled_materials), "label_class_count": len(classes), }) return rows def label_density(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> dict[str, Any]: rows = formula_character_rows(records, labels) by_genre: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in rows: by_genre[row["genre"]].append(row) per_genre = {} for genre, genre_rows in sorted(by_genre.items()): per_genre[genre] = { "labelled_formulas": len(genre_rows), "avg_distinct_classes_per_labelled_formula": sum(r["label_class_count"] for r in genre_rows) / len(genre_rows), "avg_labelled_materials_per_labelled_formula": sum(r["labelled_material_count"] for r in genre_rows) / len(genre_rows), "formulas_with_ge_2_classes": sum(1 for r in genre_rows if r["label_class_count"] >= 2), "formulas_with_ge_3_classes": sum(1 for r in genre_rows if r["label_class_count"] >= 3), } return { "axis": axis_name, "labelled_formulas": len(rows), "avg_distinct_classes_per_labelled_formula": sum(r["label_class_count"] for r in rows) / len(rows) if rows else None, "avg_labelled_materials_per_labelled_formula": sum(r["labelled_material_count"] for r in rows) / len(rows) if rows else None, "per_genre": per_genre, } def cosine_similarity(left: dict[str, float], right: dict[str, float], codes: list[str]) -> float | None: a = [left.get(code, 0.0) for code in codes] b = [right.get(code, 0.0) for code in codes] left_norm = math.sqrt(sum(v * v for v in a)) right_norm = math.sqrt(sum(v * v for v in b)) if not left_norm or not right_norm: return None return sum(x * y for x, y in zip(a, b)) / (left_norm * right_norm) def js_divergence(left: dict[str, float], right: dict[str, float], codes: list[str]) -> float | None: left_total = sum(left.get(code, 0.0) for code in codes) right_total = sum(right.get(code, 0.0) for code in codes) if not left_total or not right_total: return None p = [left.get(code, 0.0) / left_total for code in codes] q = [right.get(code, 0.0) / right_total for code in codes] m = [(x + y) / 2 for x, y in zip(p, q)] def kl(a: list[float], b: list[float]) -> float: return sum(x * math.log2(x / y) for x, y in zip(a, b) if x and y) return 0.5 * kl(p, m) + 0.5 * kl(q, m) def pairwise_vector_metrics(vectors: dict[str, Counter[str]], codes: list[str]) -> dict[str, Any]: pairwise = [] genres = sorted(vectors) for i, left in enumerate(genres): for right in genres[i + 1:]: cosine = cosine_similarity(vectors[left], vectors[right], codes) jsd = js_divergence(vectors[left], vectors[right], codes) pairwise.append({ "left": left, "right": right, "cosine_similarity": cosine, "cosine_distance": None if cosine is None else 1.0 - cosine, "js_divergence_bits": jsd, }) cosines = [p["cosine_similarity"] for p in pairwise if p["cosine_similarity"] is not None] cosine_distances = [p["cosine_distance"] for p in pairwise if p["cosine_distance"] is not None] jsds = [p["js_divergence_bits"] for p in pairwise if p["js_divergence_bits"] is not None] cosine_distance_matrix = { genre: { other: 0.0 if genre == other else None for other in genres } for genre in genres } js_divergence_matrix = { genre: { other: 0.0 if genre == other else None for other in genres } for genre in genres } for row in pairwise: left = row["left"] right = row["right"] cosine_distance_matrix[left][right] = row["cosine_distance"] cosine_distance_matrix[right][left] = row["cosine_distance"] js_divergence_matrix[left][right] = row["js_divergence_bits"] js_divergence_matrix[right][left] = row["js_divergence_bits"] return { "pairwise": pairwise, "mean_pairwise_cosine_similarity": sum(cosines) / len(cosines) if cosines else None, "mean_pairwise_cosine_distance": sum(cosine_distances) / len(cosine_distances) if cosine_distances else None, "min_pairwise_cosine_similarity": min(cosines) if cosines else None, "max_pairwise_cosine_similarity": max(cosines) if cosines else None, "mean_pairwise_js_divergence_bits": sum(jsds) / len(jsds) if jsds else None, "max_pairwise_js_divergence_bits": max(jsds) if jsds else None, "cosine_distance_matrix": cosine_distance_matrix, "js_divergence_bits_matrix": js_divergence_matrix, } def distributional_probe( records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str, min_classes_per_formula: int = 1, weighted: bool = False, ) -> dict[str, Any]: rows = [row for row in formula_character_rows(records, labels) if row["label_class_count"] >= min_classes_per_formula] codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}}) vectors: dict[str, Counter[str]] = defaultdict(Counter) kept_by_genre = Counter() for row in rows: kept_by_genre[row["genre"]] += 1 vectors[row["genre"]].update(row["weighted_classes"] if weighted else row["material_classes"]) metrics = pairwise_vector_metrics(vectors, codes) return { "method": ( f"per-genre class-count vector cosine/JS over {axis_name}; " f"formulas require >= {min_classes_per_formula} distinct character classes; " f"{'label-confidence weighted' if weighted else 'material occurrence counted'}" ), "kept_labelled_formulas_by_genre": dict(sorted(kept_by_genre.items())), "class_count_vectors_by_genre": { genre: {code: vectors[genre].get(code, 0) for code in codes if vectors[genre].get(code, 0)} for genre in sorted(vectors) }, **metrics, } def prevalence_distributional_probe( records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str, ) -> dict[str, Any]: """Unthresholded class prevalence profile by genre. Each vector entry is the fraction of labelled formulas in that genre containing the class. Magnitudes are retained; no active-set thresholding is applied. """ rows = formula_character_rows(records, labels) codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}}) by_genre: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in rows: by_genre[row["genre"]].append(row) vectors: dict[str, Counter[str]] = defaultdict(Counter) kept_by_genre = {} for genre, genre_rows in sorted(by_genre.items()): kept_by_genre[genre] = len(genre_rows) counts = Counter(code for row in genre_rows for code in row["classes"]) for code in codes: vectors[genre][code] = counts.get(code, 0) / len(genre_rows) return { "method": ( f"per-genre unthresholded class-prevalence vectors over {axis_name}; " "entry = fraction of labelled formulas in the genre containing the class" ), "kept_labelled_formulas_by_genre": kept_by_genre, "class_prevalence_vectors_by_genre": { genre: {code: vectors[genre].get(code, 0.0) for code in codes} for genre in sorted(vectors) }, **pairwise_vector_metrics(vectors, codes), } def material_level_probe(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> dict[str, Any]: codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}}) material_classes_by_genre: dict[str, dict[str, set[str]]] = defaultdict(dict) for record in records: if record.get("is_control"): continue genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown" for comp in census.formula_components(record): cas = str(comp.get("cas")) label = labels.get(cas) if not label: continue classes = {code for code in label_codes(label) if code not in {"Z", "carrier"}} if classes: material_classes_by_genre[genre][cas] = classes presence_profiles = { genre: sorted({code for classes in materials.values() for code in classes}) for genre, materials in material_classes_by_genre.items() } pairwise_jaccard = [] genres = sorted(presence_profiles) for i, left in enumerate(genres): for right in genres[i + 1:]: a, b = set(presence_profiles[left]), set(presence_profiles[right]) pairwise_jaccard.append({ "left": left, "right": right, "jaccard": len(a & b) / max(1, len(a | b)), "intersection": sorted(a & b), "union": sorted(a | b), }) vectors = {} for genre, materials in material_classes_by_genre.items(): vectors[genre] = Counter(code for classes in materials.values() for code in classes) metrics = pairwise_vector_metrics(vectors, codes) return { "method": f"unique-material-level character overlap for {axis_name}, not formula-level aggregation", "unique_labelled_materials_by_genre": {genre: len(materials) for genre, materials in sorted(material_classes_by_genre.items())}, "presence_profiles_by_genre": dict(sorted(presence_profiles.items())), "pairwise_presence_jaccard": pairwise_jaccard, "mean_pairwise_presence_jaccard": ( sum(p["jaccard"] for p in pairwise_jaccard) / len(pairwise_jaccard) if pairwise_jaccard else None ), "class_count_vectors_by_genre": { genre: {code: vectors[genre].get(code, 0) for code in codes if vectors[genre].get(code, 0)} for genre in sorted(vectors) }, **metrics, } def discrimination_probe( records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str, threshold: float = 0.05, ) -> dict[str, Any]: by_genre = formula_character_sets(records, labels) genre_profiles: dict[str, set[str]] = {} prevalence_by_genre = {} for genre, sets in sorted(by_genre.items()): if len(sets) < 10: continue counts = Counter(code for s in sets for code in s) prevalence_by_genre[genre] = {code: count / len(sets) for code, count in sorted(counts.items())} genre_profiles[genre] = {code for code, count in counts.items() if count / len(sets) >= threshold} pairwise = [] genres = sorted(genre_profiles) for i, left in enumerate(genres): for right in genres[i + 1:]: a, b = genre_profiles[left], genre_profiles[right] pairwise.append({ "left": left, "right": right, "jaccard": len(a & b) / max(1, len(a | b)), "intersection": sorted(a & b), "union": sorted(a | b), }) mean_jaccard = sum(p["jaccard"] for p in pairwise) / len(pairwise) if pairwise else None return { "method": ( f"per-genre formula-level set Jaccard over {axis_name}; " f"class active if present in >={threshold:.0%} of labelled formulas for genre" ), "activity_threshold": threshold, "formula_labelled_by_genre": {k: len(v) for k, v in by_genre.items()}, "class_prevalence_by_genre": prevalence_by_genre, "genre_profiles": {k: sorted(v) for k, v in genre_profiles.items()}, "pairwise": pairwise, "mean_pairwise_jaccard": mean_jaccard, } def threshold_sensitivity_probe(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> list[dict[str, Any]]: rows = [] for threshold in (0.05, 0.10, 0.15): probe = discrimination_probe(records, labels, axis_name, threshold=threshold) rows.append({ "threshold": threshold, "mean_pairwise_jaccard": probe["mean_pairwise_jaccard"], "genre_profiles": probe["genre_profiles"], "prevalence_by_genre_before_thresholding": probe["class_prevalence_by_genre"], }) return rows def evaluate_snapshot(name: str, path: Path, records: list[dict[str, Any]]) -> dict[str, Any]: rows = load_jsonl(path) labels = {row["cas"]: row for row in rows} base = discrimination_probe(records, labels, "26-class character labels") base_density = label_density(records, labels, "26-class character labels") base_distribution = distributional_probe(records, labels, "26-class character labels") base_prevalence_distribution = prevalence_distributional_probe(records, labels, "26-class character labels") base_well_labelled_distribution = distributional_probe(records, labels, "26-class character labels", min_classes_per_formula=3) base_confidence_distribution = distributional_probe(records, labels, "26-class character labels", min_classes_per_formula=2, weighted=True) base_material = material_level_probe(records, labels, "26-class character labels") group_results = {} for grouping_name, spec in GROUPINGS.items(): grouped = transform_labels(labels, spec["mapping"]) discrimination_precheck = discrimination_probe(records, grouped, grouping_name) group_results[grouping_name] = { "justification": spec["justification"], "n_superclasses": len({v for v in spec["mapping"].values() if v != "carrier"}), "class_to_superclass": spec["mapping"], "label_density": label_density(records, grouped, grouping_name), "discrimination_precheck": discrimination_precheck, "superclass_prevalence_by_genre": discrimination_precheck["class_prevalence_by_genre"], "threshold_sensitivity": threshold_sensitivity_probe(records, grouped, grouping_name), "material_level_probe": material_level_probe(records, grouped, grouping_name), "distributional_probe": distributional_probe(records, grouped, grouping_name), "prevalence_distributional_metric": prevalence_distributional_probe(records, grouped, grouping_name), "well_labelled_distributional_probe_ge3": distributional_probe(records, grouped, grouping_name, min_classes_per_formula=3), "confidence_weighted_distributional_probe_ge2": distributional_probe(records, grouped, grouping_name, min_classes_per_formula=2, weighted=True), } best_group_name = min( group_results, key=lambda g: group_results[g]["discrimination_precheck"]["mean_pairwise_jaccard"] if group_results[g]["discrimination_precheck"]["mean_pairwise_jaccard"] is not None else float("inf"), ) best_j = group_results[best_group_name]["discrimination_precheck"]["mean_pairwise_jaccard"] base_j = base["mean_pairwise_jaccard"] material_improvement = base_j is not None and best_j is not None and best_j <= base_j - 0.05 return { "label_snapshot": name, "label_path": str(path.relative_to(ROOT)), "label_count": len(rows), "label_density_26_class": base_density, "twenty_six_class": base, "material_level_probe_26_class": base_material, "distributional_probe_26_class": base_distribution, "prevalence_distributional_metric_26_class": base_prevalence_distribution, "well_labelled_distributional_probe_26_class_ge3": base_well_labelled_distribution, "confidence_weighted_distributional_probe_26_class_ge2": base_confidence_distribution, "groupings": group_results, "best_grouping": best_group_name if material_improvement else None, "lowest_jaccard_grouping": best_group_name, "best_grouping_mean_pairwise_jaccard": best_j, "materially_better_than_26_class": material_improvement, } def recommendation(primary: dict[str, Any]) -> str: base_j = primary["twenty_six_class"]["mean_pairwise_jaccard"] best_j = primary["best_grouping_mean_pairwise_jaccard"] base_distance = primary["prevalence_distributional_metric_26_class"]["mean_pairwise_cosine_distance"] base_jsd = primary["prevalence_distributional_metric_26_class"]["mean_pairwise_js_divergence_bits"] return ( "Retract the superclass presence-Jaccard conclusion: the 1.000 is a collapse/presence artifact, " f"not proof that character is dead. Keep corrected distributional metrics as the circuit-breaker. " f"Under unthresholded prevalence vectors, 26-class character shows weak but nonzero profile " f"separation (mean cosine distance {base_distance:.3f}; mean JS divergence {base_jsd:.3f} bits; " f"presence-Jaccard baseline {base_j:.3f}, best collapsed presence-Jaccard {best_j:.3f}). " "Recommended direction: character may resume only at 26-class granularity under distributional " "and well-labelled filters; do not use superclass presence-Jaccard for go/no-go decisions." ) def character_verdict(primary: dict[str, Any]) -> dict[str, Any]: metric = primary["prevalence_distributional_metric_26_class"] return { "verdict": "discriminative-weak", "basis": "26-class unthresholded prevalence distributional metric", "mean_pairwise_cosine_distance": metric["mean_pairwise_cosine_distance"], "mean_pairwise_js_divergence_bits": metric["mean_pairwise_js_divergence_bits"], "interpretation": ( "Character profiles are weakly but genuinely separated at 26-class granularity. " "Superclass collapse remains non-discriminative and threshold-driven. Character may resume " "only as a secondary axis gated by this distributional metric; presence-Jaccard must not be " "used again as the circuit-breaker." ), "next_action_policy": "present_for_human_decision_no_training_no_upload_no_sourcing_no_label_expansion", } def matrix_markdown(title: str, matrix: dict[str, dict[str, float | None]]) -> list[str]: genres = sorted(matrix) rows = [ f"### {title}", "", "| Genre | " + " | ".join(f"`{genre}`" for genre in genres) + " |", "| --- | " + " | ".join("---:" for _ in genres) + " |", ] for genre in genres: values = [] for other in genres: value = matrix[genre][other] values.append("" if value is None else f"{value:.3f}") rows.append(f"| `{genre}` | " + " | ".join(values) + " |") rows.append("") return rows def write_markdown(report: dict[str, Any]) -> None: primary = report["primary_snapshot_result"] rows = [] base_j = primary["twenty_six_class"]["mean_pairwise_jaccard"] base_cos = primary["distributional_probe_26_class"]["mean_pairwise_cosine_similarity"] base_ge3_cos = primary["well_labelled_distributional_probe_26_class_ge3"]["mean_pairwise_cosine_similarity"] base_prevalence = primary["prevalence_distributional_metric_26_class"] verdict = report["character_verdict"] rows.append(f"| 26-class | 26 | {base_j:.3f} | baseline |") for name, result in primary["groupings"].items(): j = result["discrimination_precheck"]["mean_pairwise_jaccard"] delta = j - base_j rows.append(f"| {name} | {result['n_superclasses']} | {j:.3f} | {delta:+.3f} |") density = primary["label_density_26_class"] density_rows = [ "| 26-class | " f"{density['avg_distinct_classes_per_labelled_formula']:.2f} | " f"{density['avg_labelled_materials_per_labelled_formula']:.2f} |" ] for name, result in primary["groupings"].items(): d = result["label_density"] density_rows.append( f"| {name} | {d['avg_distinct_classes_per_labelled_formula']:.2f} | " f"{d['avg_labelled_materials_per_labelled_formula']:.2f} |" ) superclass_sets = [] for name, result in primary["groupings"].items(): profiles = result["discrimination_precheck"]["genre_profiles"] superclass_sets.append(f"### {name}") for genre, classes in sorted(profiles.items()): superclass_sets.append(f"- `{genre}`: {', '.join(classes)}") superclass_sets.append("") metric_rows = [ "| 26-class | " f"{primary['distributional_probe_26_class']['mean_pairwise_cosine_similarity']:.3f} | " f"{primary['well_labelled_distributional_probe_26_class_ge3']['mean_pairwise_cosine_similarity']:.3f} | " f"{primary['material_level_probe_26_class']['mean_pairwise_cosine_similarity']:.3f} |", ] for name, result in primary["groupings"].items(): metric_rows.append( f"| {name} | " f"{result['distributional_probe']['mean_pairwise_cosine_similarity']:.3f} | " f"{result['well_labelled_distributional_probe_ge3']['mean_pairwise_cosine_similarity']:.3f} | " f"{result['material_level_probe']['mean_pairwise_cosine_similarity']:.3f} |" ) threshold_rows = [] for name, result in primary["groupings"].items(): for row in result["threshold_sensitivity"]: threshold_rows.append(f"| {name} | {row['threshold']:.0%} | {row['mean_pairwise_jaccard']:.3f} |") prevalence_metric_rows = [ "| 26-class | 26 | " f"{base_prevalence['mean_pairwise_cosine_distance']:.3f} | " f"{base_prevalence['mean_pairwise_js_divergence_bits']:.3f} |" ] for name, result in primary["groupings"].items(): metric = result["prevalence_distributional_metric"] prevalence_metric_rows.append( f"| {name} | {result['n_superclasses']} | " f"{metric['mean_pairwise_cosine_distance']:.3f} | " f"{metric['mean_pairwise_js_divergence_bits']:.3f} |" ) distributional_matrices = [ "## Unthresholded Prevalence Distributional Metric", "", "This is the corrected character discrimination test. Vectors retain the raw per-genre prevalence " "of each class before any active-set thresholding; cosine is reported as distance (`1 - similarity`) " "and JS divergence is reported in bits.", "", "| Axis | classes | mean cosine distance | mean JS divergence (bits) |", "| --- | ---: | ---: | ---: |", *prevalence_metric_rows, "", *matrix_markdown( "26-class cosine distance", base_prevalence["cosine_distance_matrix"], ), *matrix_markdown( "26-class JS divergence (bits)", base_prevalence["js_divergence_bits_matrix"], ), ] for name, result in primary["groupings"].items(): metric = result["prevalence_distributional_metric"] distributional_matrices.extend(matrix_markdown(f"{name} cosine distance", metric["cosine_distance_matrix"])) distributional_matrices.extend(matrix_markdown(f"{name} JS divergence (bits)", metric["js_divergence_bits_matrix"])) md = [ "# Character grouping diagnostic v8", "", f"Primary snapshot: `{primary['label_snapshot']}` (`{primary['label_path']}`).", "", "## Verdict", "", f"**Character verdict: {verdict['verdict']}.** Under the corrected 26-class unthresholded " f"prevalence metric, mean cosine distance is {verdict['mean_pairwise_cosine_distance']:.3f} " f"and mean JS divergence is {verdict['mean_pairwise_js_divergence_bits']:.3f} bits. This supports " "weak but genuine distributional separation, suitable only as a secondary axis pending human decision.", "", "**Artifact.** The perfect superclass Jaccard is mechanical: 1.000 = all genres have identical " "thresholded superclass active sets, NOT improved discrimination. After collapse, every genre profile " "contains every available superclass, so set-presence Jaccard has no room to differ. Distributional " f"class-count metrics are not perfect (26-class all-labelled mean cosine {base_cos:.3f}; " f"26-class >=3-label mean cosine {base_ge3_cos:.3f}), so the prior character-is-non-discriminative " "verdict is retracted.", "", "| Axis | classes | mean pairwise Jaccard | delta vs 26 |", "| --- | ---: | ---: | ---: |", *rows, "", "## Superclass Sets Compared", "", *superclass_sets, "## Label Density", "", "| Axis | avg distinct classes / labelled formula | avg labelled materials / labelled formula |", "| --- | ---: | ---: |", *density_rows, "", "## Corrected Distributional Metrics", "", "| Axis | mean cosine, all labelled | mean cosine, >=3 classes | material-level mean cosine |", "| --- | ---: | ---: | ---: |", *metric_rows, "", "Lower cosine means more genre-profile separation. The 26-class axis keeps the strongest separation; " "collapsing to superclasses weakens it and made presence-Jaccard unusable. `best_grouping` is null " "unless a grouped axis materially improves on 26-class; the lowest grouped Jaccard is only recorded " "as `lowest_jaccard_grouping` for audit.", "", *distributional_matrices, "## Threshold Sensitivity", "", "| Grouping | activity threshold | mean pairwise Jaccard |", "| --- | ---: | ---: |", *threshold_rows, "", f"Recommendation: **{report['recommendation']}**", "", f"Character verdict policy: **{verdict['interpretation']}**", "", "Substantivity remains confirmed and publishable independent of this character diagnostic. " "Next human decision: ship substantivity, resume character only under the corrected 26-class " "distributional circuit-breaker, or park character for a trajectory-proportion pivot.", "", "No new sourcing, label expansion, dataset assembly, training, or upload was performed.", ] OUT_MD.write_text("\n".join(md) + "\n", encoding="utf-8") def main() -> None: records = census.load_formula_records() snapshots = { name: evaluate_snapshot(name, path, records) for name, path in LABEL_SNAPSHOTS.items() if path.exists() } primary_name = "v8_2_clause_guard_dominance" primary = snapshots[primary_name] report = { "pimt_version": "v8", "diagnostic": "character_grouping_existing_labels_only", "primary_snapshot": primary_name, "primary_snapshot_result": primary, "character_verdict": character_verdict(primary), "all_snapshot_results": snapshots, "recommendation": recommendation(primary), "stop": "NO_NEW_SOURCING_NO_LABEL_EXPANSION_NO_DATASET_ASSEMBLY_NO_TRAINING_NO_HF_UPLOAD", } OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") write_markdown(report) print(json.dumps({ "primary_snapshot": primary_name, "twenty_six_class_jaccard": primary["twenty_six_class"]["mean_pairwise_jaccard"], "best_grouping": primary["best_grouping"], "best_grouping_jaccard": primary["best_grouping_mean_pairwise_jaccard"], "materially_better_than_26_class": primary["materially_better_than_26_class"], "recommendation": report["recommendation"], }, indent=2, sort_keys=True)) if __name__ == "__main__": main()