| |
| """Character superclass grouping diagnostic for existing labels only. |
| |
| Diagnostics only: no new sourcing, no label expansion, no dataset assembly, |
| no training, and no upload. |
| """ |
| from __future__ import annotations |
|
|
| from collections import Counter, defaultdict |
| import json |
| import math |
| from pathlib import Path |
| import sys |
| from typing import Any |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| DATA = ROOT / "data" |
| ARTIFACTS = ROOT / "artifacts" |
| OUT_JSON = ARTIFACTS / "character_grouping_diagnostic_v8.json" |
| OUT_MD = ARTIFACTS / "character_grouping_diagnostic_v8.md" |
|
|
| sys.path.insert(0, str(ROOT / "scripts")) |
| import build_odour_taxonomy_census_v8 as census |
|
|
|
|
| LABEL_SNAPSHOTS = { |
| "v8_2": DATA / "odour_character_labels_v8_2.jsonl", |
| "v8_2_clause_guard": DATA / "odour_character_labels_v8_2_clause_guard.jsonl", |
| "v8_2_clause_guard_dominance": DATA / "odour_character_labels_v8_2_clause_guard_dominance.jsonl", |
| } |
|
|
|
|
| GROUPINGS: dict[str, dict[str, Any]] = { |
| "six_superclass": { |
| "justification": ( |
| "Collapses the 26-letter taxonomy into perfumery families: fresh/top-note, floral, " |
| "edible/gourmand, balsamic/woody/fixative, warm/spicy/animalic/smoky, and solvent carriers." |
| ), |
| "mapping": { |
| "A": "fresh_citrus_green", |
| "B": "fresh_citrus_green", |
| "C": "fresh_citrus_green", |
| "F": "fresh_citrus_green", |
| "G": "fresh_citrus_green", |
| "H": "fresh_citrus_green", |
| "K": "fresh_citrus_green", |
| "L": "floral", |
| "I": "floral", |
| "J": "floral", |
| "M": "floral", |
| "N": "floral", |
| "O": "floral", |
| "R": "floral", |
| "D": "edible_gourmand", |
| "E": "edible_gourmand", |
| "P": "edible_gourmand", |
| "V": "edible_gourmand", |
| "Q": "balsamic_woody_mossy", |
| "W": "balsamic_woody_mossy", |
| "X": "balsamic_woody_mossy", |
| "Y": "balsamic_woody_mossy", |
| "S": "animalic_spicy_smoky", |
| "T": "animalic_spicy_smoky", |
| "U": "animalic_spicy_smoky", |
| "Z": "carrier", |
| }, |
| }, |
| "eight_superclass": { |
| "justification": ( |
| "Keeps fruit/citrus/green fresh notes separate from aromatic cool herbs, splits gourmand " |
| "from phenolic/savory edible, and keeps musk/animal/smoke separate from spice." |
| ), |
| "mapping": { |
| "A": "fresh_citrus_green_fruit", |
| "C": "fresh_citrus_green_fruit", |
| "F": "fresh_citrus_green_fruit", |
| "G": "fresh_citrus_green_fruit", |
| "B": "aromatic_cool_herbal", |
| "H": "aromatic_cool_herbal", |
| "K": "aromatic_cool_herbal", |
| "I": "floral", |
| "J": "floral", |
| "L": "floral", |
| "M": "floral", |
| "N": "floral", |
| "O": "floral", |
| "R": "floral", |
| "D": "gourmand_sweet", |
| "V": "gourmand_sweet", |
| "E": "savory_phenolic", |
| "P": "savory_phenolic", |
| "Q": "balsamic_woody_mossy", |
| "W": "balsamic_woody_mossy", |
| "Y": "balsamic_woody_mossy", |
| "S": "spice", |
| "T": "animalic_musk_smoke", |
| "U": "animalic_musk_smoke", |
| "X": "animalic_musk_smoke", |
| "Z": "carrier", |
| }, |
| }, |
| } |
|
|
|
|
| def load_jsonl(path: Path) -> list[dict[str, Any]]: |
| return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] |
|
|
|
|
| def label_codes(label: dict[str, Any]) -> set[str]: |
| codes = label.get("class_codes") |
| if isinstance(codes, list): |
| return {str(code) for code in codes} |
| code = label.get("class_code") |
| return {str(code)} if code else set() |
|
|
|
|
| def label_confidence(label: dict[str, Any]) -> float: |
| """Return available label confidence, defaulting to 1 for source-derived labels.""" |
| for key in ("confidence", "label_confidence", "score"): |
| value = label.get(key) |
| if isinstance(value, int | float): |
| return float(value) |
| return 1.0 |
|
|
|
|
| def transform_labels(labels: dict[str, dict[str, Any]], mapping: dict[str, str]) -> dict[str, dict[str, Any]]: |
| out = {} |
| for cas, label in labels.items(): |
| super_codes = sorted({mapping[code] for code in label_codes(label) if code in mapping and mapping[code] != "carrier"}) |
| if not super_codes: |
| continue |
| out[cas] = { |
| **label, |
| "class_code": super_codes[0], |
| "class_codes": super_codes, |
| "original_class_codes": sorted(label_codes(label)), |
| "grouping": "superclass", |
| } |
| return out |
|
|
|
|
| def formula_character_sets(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]]) -> dict[str, list[set[str]]]: |
| by_genre: dict[str, list[set[str]]] = defaultdict(list) |
| for record in records: |
| if record.get("is_control"): |
| continue |
| genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown" |
| chars = set() |
| for comp in census.formula_components(record): |
| label = labels.get(str(comp.get("cas"))) |
| if label: |
| chars.update(code for code in label_codes(label) if code != "Z" and code != "carrier") |
| if chars: |
| by_genre[genre].append(chars) |
| return by_genre |
|
|
|
|
| def formula_character_rows(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: |
| rows = [] |
| for record in records: |
| if record.get("is_control"): |
| continue |
| genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown" |
| formula_id = record.get("formula_id") or record.get("metadata", {}).get("formula_id") |
| classes: set[str] = set() |
| weighted_classes: Counter[str] = Counter() |
| material_classes: Counter[str] = Counter() |
| labelled_materials = set() |
| for comp in census.formula_components(record): |
| cas = str(comp.get("cas")) |
| label = labels.get(cas) |
| if not label: |
| continue |
| codes = {code for code in label_codes(label) if code not in {"Z", "carrier"}} |
| if not codes: |
| continue |
| labelled_materials.add(cas) |
| confidence = label_confidence(label) |
| for code in codes: |
| classes.add(code) |
| weighted_classes[code] += confidence |
| material_classes[code] += 1 |
| if classes: |
| rows.append({ |
| "genre": genre, |
| "formula_id": formula_id, |
| "classes": classes, |
| "weighted_classes": weighted_classes, |
| "material_classes": material_classes, |
| "labelled_material_count": len(labelled_materials), |
| "label_class_count": len(classes), |
| }) |
| return rows |
|
|
|
|
| def label_density(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> dict[str, Any]: |
| rows = formula_character_rows(records, labels) |
| by_genre: dict[str, list[dict[str, Any]]] = defaultdict(list) |
| for row in rows: |
| by_genre[row["genre"]].append(row) |
| per_genre = {} |
| for genre, genre_rows in sorted(by_genre.items()): |
| per_genre[genre] = { |
| "labelled_formulas": len(genre_rows), |
| "avg_distinct_classes_per_labelled_formula": sum(r["label_class_count"] for r in genre_rows) / len(genre_rows), |
| "avg_labelled_materials_per_labelled_formula": sum(r["labelled_material_count"] for r in genre_rows) / len(genre_rows), |
| "formulas_with_ge_2_classes": sum(1 for r in genre_rows if r["label_class_count"] >= 2), |
| "formulas_with_ge_3_classes": sum(1 for r in genre_rows if r["label_class_count"] >= 3), |
| } |
| return { |
| "axis": axis_name, |
| "labelled_formulas": len(rows), |
| "avg_distinct_classes_per_labelled_formula": sum(r["label_class_count"] for r in rows) / len(rows) if rows else None, |
| "avg_labelled_materials_per_labelled_formula": sum(r["labelled_material_count"] for r in rows) / len(rows) if rows else None, |
| "per_genre": per_genre, |
| } |
|
|
|
|
| def cosine_similarity(left: dict[str, float], right: dict[str, float], codes: list[str]) -> float | None: |
| a = [left.get(code, 0.0) for code in codes] |
| b = [right.get(code, 0.0) for code in codes] |
| left_norm = math.sqrt(sum(v * v for v in a)) |
| right_norm = math.sqrt(sum(v * v for v in b)) |
| if not left_norm or not right_norm: |
| return None |
| return sum(x * y for x, y in zip(a, b)) / (left_norm * right_norm) |
|
|
|
|
| def js_divergence(left: dict[str, float], right: dict[str, float], codes: list[str]) -> float | None: |
| left_total = sum(left.get(code, 0.0) for code in codes) |
| right_total = sum(right.get(code, 0.0) for code in codes) |
| if not left_total or not right_total: |
| return None |
| p = [left.get(code, 0.0) / left_total for code in codes] |
| q = [right.get(code, 0.0) / right_total for code in codes] |
| m = [(x + y) / 2 for x, y in zip(p, q)] |
|
|
| def kl(a: list[float], b: list[float]) -> float: |
| return sum(x * math.log2(x / y) for x, y in zip(a, b) if x and y) |
|
|
| return 0.5 * kl(p, m) + 0.5 * kl(q, m) |
|
|
|
|
| def pairwise_vector_metrics(vectors: dict[str, Counter[str]], codes: list[str]) -> dict[str, Any]: |
| pairwise = [] |
| genres = sorted(vectors) |
| for i, left in enumerate(genres): |
| for right in genres[i + 1:]: |
| cosine = cosine_similarity(vectors[left], vectors[right], codes) |
| jsd = js_divergence(vectors[left], vectors[right], codes) |
| pairwise.append({ |
| "left": left, |
| "right": right, |
| "cosine_similarity": cosine, |
| "cosine_distance": None if cosine is None else 1.0 - cosine, |
| "js_divergence_bits": jsd, |
| }) |
| cosines = [p["cosine_similarity"] for p in pairwise if p["cosine_similarity"] is not None] |
| cosine_distances = [p["cosine_distance"] for p in pairwise if p["cosine_distance"] is not None] |
| jsds = [p["js_divergence_bits"] for p in pairwise if p["js_divergence_bits"] is not None] |
| cosine_distance_matrix = { |
| genre: { |
| other: 0.0 if genre == other else None |
| for other in genres |
| } |
| for genre in genres |
| } |
| js_divergence_matrix = { |
| genre: { |
| other: 0.0 if genre == other else None |
| for other in genres |
| } |
| for genre in genres |
| } |
| for row in pairwise: |
| left = row["left"] |
| right = row["right"] |
| cosine_distance_matrix[left][right] = row["cosine_distance"] |
| cosine_distance_matrix[right][left] = row["cosine_distance"] |
| js_divergence_matrix[left][right] = row["js_divergence_bits"] |
| js_divergence_matrix[right][left] = row["js_divergence_bits"] |
| return { |
| "pairwise": pairwise, |
| "mean_pairwise_cosine_similarity": sum(cosines) / len(cosines) if cosines else None, |
| "mean_pairwise_cosine_distance": sum(cosine_distances) / len(cosine_distances) if cosine_distances else None, |
| "min_pairwise_cosine_similarity": min(cosines) if cosines else None, |
| "max_pairwise_cosine_similarity": max(cosines) if cosines else None, |
| "mean_pairwise_js_divergence_bits": sum(jsds) / len(jsds) if jsds else None, |
| "max_pairwise_js_divergence_bits": max(jsds) if jsds else None, |
| "cosine_distance_matrix": cosine_distance_matrix, |
| "js_divergence_bits_matrix": js_divergence_matrix, |
| } |
|
|
|
|
| def distributional_probe( |
| records: list[dict[str, Any]], |
| labels: dict[str, dict[str, Any]], |
| axis_name: str, |
| min_classes_per_formula: int = 1, |
| weighted: bool = False, |
| ) -> dict[str, Any]: |
| rows = [row for row in formula_character_rows(records, labels) if row["label_class_count"] >= min_classes_per_formula] |
| codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}}) |
| vectors: dict[str, Counter[str]] = defaultdict(Counter) |
| kept_by_genre = Counter() |
| for row in rows: |
| kept_by_genre[row["genre"]] += 1 |
| vectors[row["genre"]].update(row["weighted_classes"] if weighted else row["material_classes"]) |
| metrics = pairwise_vector_metrics(vectors, codes) |
| return { |
| "method": ( |
| f"per-genre class-count vector cosine/JS over {axis_name}; " |
| f"formulas require >= {min_classes_per_formula} distinct character classes; " |
| f"{'label-confidence weighted' if weighted else 'material occurrence counted'}" |
| ), |
| "kept_labelled_formulas_by_genre": dict(sorted(kept_by_genre.items())), |
| "class_count_vectors_by_genre": { |
| genre: {code: vectors[genre].get(code, 0) for code in codes if vectors[genre].get(code, 0)} |
| for genre in sorted(vectors) |
| }, |
| **metrics, |
| } |
|
|
|
|
| def prevalence_distributional_probe( |
| records: list[dict[str, Any]], |
| labels: dict[str, dict[str, Any]], |
| axis_name: str, |
| ) -> dict[str, Any]: |
| """Unthresholded class prevalence profile by genre. |
| |
| Each vector entry is the fraction of labelled formulas in that genre |
| containing the class. Magnitudes are retained; no active-set thresholding is |
| applied. |
| """ |
| rows = formula_character_rows(records, labels) |
| codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}}) |
| by_genre: dict[str, list[dict[str, Any]]] = defaultdict(list) |
| for row in rows: |
| by_genre[row["genre"]].append(row) |
|
|
| vectors: dict[str, Counter[str]] = defaultdict(Counter) |
| kept_by_genre = {} |
| for genre, genre_rows in sorted(by_genre.items()): |
| kept_by_genre[genre] = len(genre_rows) |
| counts = Counter(code for row in genre_rows for code in row["classes"]) |
| for code in codes: |
| vectors[genre][code] = counts.get(code, 0) / len(genre_rows) |
|
|
| return { |
| "method": ( |
| f"per-genre unthresholded class-prevalence vectors over {axis_name}; " |
| "entry = fraction of labelled formulas in the genre containing the class" |
| ), |
| "kept_labelled_formulas_by_genre": kept_by_genre, |
| "class_prevalence_vectors_by_genre": { |
| genre: {code: vectors[genre].get(code, 0.0) for code in codes} |
| for genre in sorted(vectors) |
| }, |
| **pairwise_vector_metrics(vectors, codes), |
| } |
|
|
|
|
| def material_level_probe(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> dict[str, Any]: |
| codes = sorted({code for label in labels.values() for code in label_codes(label) if code not in {"Z", "carrier"}}) |
| material_classes_by_genre: dict[str, dict[str, set[str]]] = defaultdict(dict) |
| for record in records: |
| if record.get("is_control"): |
| continue |
| genre = record.get("genre") or record.get("metadata", {}).get("generation_strategy") or "unknown" |
| for comp in census.formula_components(record): |
| cas = str(comp.get("cas")) |
| label = labels.get(cas) |
| if not label: |
| continue |
| classes = {code for code in label_codes(label) if code not in {"Z", "carrier"}} |
| if classes: |
| material_classes_by_genre[genre][cas] = classes |
| presence_profiles = { |
| genre: sorted({code for classes in materials.values() for code in classes}) |
| for genre, materials in material_classes_by_genre.items() |
| } |
| pairwise_jaccard = [] |
| genres = sorted(presence_profiles) |
| for i, left in enumerate(genres): |
| for right in genres[i + 1:]: |
| a, b = set(presence_profiles[left]), set(presence_profiles[right]) |
| pairwise_jaccard.append({ |
| "left": left, |
| "right": right, |
| "jaccard": len(a & b) / max(1, len(a | b)), |
| "intersection": sorted(a & b), |
| "union": sorted(a | b), |
| }) |
| vectors = {} |
| for genre, materials in material_classes_by_genre.items(): |
| vectors[genre] = Counter(code for classes in materials.values() for code in classes) |
| metrics = pairwise_vector_metrics(vectors, codes) |
| return { |
| "method": f"unique-material-level character overlap for {axis_name}, not formula-level aggregation", |
| "unique_labelled_materials_by_genre": {genre: len(materials) for genre, materials in sorted(material_classes_by_genre.items())}, |
| "presence_profiles_by_genre": dict(sorted(presence_profiles.items())), |
| "pairwise_presence_jaccard": pairwise_jaccard, |
| "mean_pairwise_presence_jaccard": ( |
| sum(p["jaccard"] for p in pairwise_jaccard) / len(pairwise_jaccard) if pairwise_jaccard else None |
| ), |
| "class_count_vectors_by_genre": { |
| genre: {code: vectors[genre].get(code, 0) for code in codes if vectors[genre].get(code, 0)} |
| for genre in sorted(vectors) |
| }, |
| **metrics, |
| } |
|
|
|
|
| def discrimination_probe( |
| records: list[dict[str, Any]], |
| labels: dict[str, dict[str, Any]], |
| axis_name: str, |
| threshold: float = 0.05, |
| ) -> dict[str, Any]: |
| by_genre = formula_character_sets(records, labels) |
| genre_profiles: dict[str, set[str]] = {} |
| prevalence_by_genre = {} |
| for genre, sets in sorted(by_genre.items()): |
| if len(sets) < 10: |
| continue |
| counts = Counter(code for s in sets for code in s) |
| prevalence_by_genre[genre] = {code: count / len(sets) for code, count in sorted(counts.items())} |
| genre_profiles[genre] = {code for code, count in counts.items() if count / len(sets) >= threshold} |
|
|
| pairwise = [] |
| genres = sorted(genre_profiles) |
| for i, left in enumerate(genres): |
| for right in genres[i + 1:]: |
| a, b = genre_profiles[left], genre_profiles[right] |
| pairwise.append({ |
| "left": left, |
| "right": right, |
| "jaccard": len(a & b) / max(1, len(a | b)), |
| "intersection": sorted(a & b), |
| "union": sorted(a | b), |
| }) |
| mean_jaccard = sum(p["jaccard"] for p in pairwise) / len(pairwise) if pairwise else None |
| return { |
| "method": ( |
| f"per-genre formula-level set Jaccard over {axis_name}; " |
| f"class active if present in >={threshold:.0%} of labelled formulas for genre" |
| ), |
| "activity_threshold": threshold, |
| "formula_labelled_by_genre": {k: len(v) for k, v in by_genre.items()}, |
| "class_prevalence_by_genre": prevalence_by_genre, |
| "genre_profiles": {k: sorted(v) for k, v in genre_profiles.items()}, |
| "pairwise": pairwise, |
| "mean_pairwise_jaccard": mean_jaccard, |
| } |
|
|
|
|
| def threshold_sensitivity_probe(records: list[dict[str, Any]], labels: dict[str, dict[str, Any]], axis_name: str) -> list[dict[str, Any]]: |
| rows = [] |
| for threshold in (0.05, 0.10, 0.15): |
| probe = discrimination_probe(records, labels, axis_name, threshold=threshold) |
| rows.append({ |
| "threshold": threshold, |
| "mean_pairwise_jaccard": probe["mean_pairwise_jaccard"], |
| "genre_profiles": probe["genre_profiles"], |
| "prevalence_by_genre_before_thresholding": probe["class_prevalence_by_genre"], |
| }) |
| return rows |
|
|
|
|
| def evaluate_snapshot(name: str, path: Path, records: list[dict[str, Any]]) -> dict[str, Any]: |
| rows = load_jsonl(path) |
| labels = {row["cas"]: row for row in rows} |
| base = discrimination_probe(records, labels, "26-class character labels") |
| base_density = label_density(records, labels, "26-class character labels") |
| base_distribution = distributional_probe(records, labels, "26-class character labels") |
| base_prevalence_distribution = prevalence_distributional_probe(records, labels, "26-class character labels") |
| base_well_labelled_distribution = distributional_probe(records, labels, "26-class character labels", min_classes_per_formula=3) |
| base_confidence_distribution = distributional_probe(records, labels, "26-class character labels", min_classes_per_formula=2, weighted=True) |
| base_material = material_level_probe(records, labels, "26-class character labels") |
| group_results = {} |
| for grouping_name, spec in GROUPINGS.items(): |
| grouped = transform_labels(labels, spec["mapping"]) |
| discrimination_precheck = discrimination_probe(records, grouped, grouping_name) |
| group_results[grouping_name] = { |
| "justification": spec["justification"], |
| "n_superclasses": len({v for v in spec["mapping"].values() if v != "carrier"}), |
| "class_to_superclass": spec["mapping"], |
| "label_density": label_density(records, grouped, grouping_name), |
| "discrimination_precheck": discrimination_precheck, |
| "superclass_prevalence_by_genre": discrimination_precheck["class_prevalence_by_genre"], |
| "threshold_sensitivity": threshold_sensitivity_probe(records, grouped, grouping_name), |
| "material_level_probe": material_level_probe(records, grouped, grouping_name), |
| "distributional_probe": distributional_probe(records, grouped, grouping_name), |
| "prevalence_distributional_metric": prevalence_distributional_probe(records, grouped, grouping_name), |
| "well_labelled_distributional_probe_ge3": distributional_probe(records, grouped, grouping_name, min_classes_per_formula=3), |
| "confidence_weighted_distributional_probe_ge2": distributional_probe(records, grouped, grouping_name, min_classes_per_formula=2, weighted=True), |
| } |
| best_group_name = min( |
| group_results, |
| key=lambda g: group_results[g]["discrimination_precheck"]["mean_pairwise_jaccard"] |
| if group_results[g]["discrimination_precheck"]["mean_pairwise_jaccard"] is not None |
| else float("inf"), |
| ) |
| best_j = group_results[best_group_name]["discrimination_precheck"]["mean_pairwise_jaccard"] |
| base_j = base["mean_pairwise_jaccard"] |
| material_improvement = base_j is not None and best_j is not None and best_j <= base_j - 0.05 |
| return { |
| "label_snapshot": name, |
| "label_path": str(path.relative_to(ROOT)), |
| "label_count": len(rows), |
| "label_density_26_class": base_density, |
| "twenty_six_class": base, |
| "material_level_probe_26_class": base_material, |
| "distributional_probe_26_class": base_distribution, |
| "prevalence_distributional_metric_26_class": base_prevalence_distribution, |
| "well_labelled_distributional_probe_26_class_ge3": base_well_labelled_distribution, |
| "confidence_weighted_distributional_probe_26_class_ge2": base_confidence_distribution, |
| "groupings": group_results, |
| "best_grouping": best_group_name if material_improvement else None, |
| "lowest_jaccard_grouping": best_group_name, |
| "best_grouping_mean_pairwise_jaccard": best_j, |
| "materially_better_than_26_class": material_improvement, |
| } |
|
|
|
|
| def recommendation(primary: dict[str, Any]) -> str: |
| base_j = primary["twenty_six_class"]["mean_pairwise_jaccard"] |
| best_j = primary["best_grouping_mean_pairwise_jaccard"] |
| base_distance = primary["prevalence_distributional_metric_26_class"]["mean_pairwise_cosine_distance"] |
| base_jsd = primary["prevalence_distributional_metric_26_class"]["mean_pairwise_js_divergence_bits"] |
| return ( |
| "Retract the superclass presence-Jaccard conclusion: the 1.000 is a collapse/presence artifact, " |
| f"not proof that character is dead. Keep corrected distributional metrics as the circuit-breaker. " |
| f"Under unthresholded prevalence vectors, 26-class character shows weak but nonzero profile " |
| f"separation (mean cosine distance {base_distance:.3f}; mean JS divergence {base_jsd:.3f} bits; " |
| f"presence-Jaccard baseline {base_j:.3f}, best collapsed presence-Jaccard {best_j:.3f}). " |
| "Recommended direction: character may resume only at 26-class granularity under distributional " |
| "and well-labelled filters; do not use superclass presence-Jaccard for go/no-go decisions." |
| ) |
|
|
|
|
| def character_verdict(primary: dict[str, Any]) -> dict[str, Any]: |
| metric = primary["prevalence_distributional_metric_26_class"] |
| return { |
| "verdict": "discriminative-weak", |
| "basis": "26-class unthresholded prevalence distributional metric", |
| "mean_pairwise_cosine_distance": metric["mean_pairwise_cosine_distance"], |
| "mean_pairwise_js_divergence_bits": metric["mean_pairwise_js_divergence_bits"], |
| "interpretation": ( |
| "Character profiles are weakly but genuinely separated at 26-class granularity. " |
| "Superclass collapse remains non-discriminative and threshold-driven. Character may resume " |
| "only as a secondary axis gated by this distributional metric; presence-Jaccard must not be " |
| "used again as the circuit-breaker." |
| ), |
| "next_action_policy": "present_for_human_decision_no_training_no_upload_no_sourcing_no_label_expansion", |
| } |
|
|
|
|
| def matrix_markdown(title: str, matrix: dict[str, dict[str, float | None]]) -> list[str]: |
| genres = sorted(matrix) |
| rows = [ |
| f"### {title}", |
| "", |
| "| Genre | " + " | ".join(f"`{genre}`" for genre in genres) + " |", |
| "| --- | " + " | ".join("---:" for _ in genres) + " |", |
| ] |
| for genre in genres: |
| values = [] |
| for other in genres: |
| value = matrix[genre][other] |
| values.append("" if value is None else f"{value:.3f}") |
| rows.append(f"| `{genre}` | " + " | ".join(values) + " |") |
| rows.append("") |
| return rows |
|
|
|
|
| def write_markdown(report: dict[str, Any]) -> None: |
| primary = report["primary_snapshot_result"] |
| rows = [] |
| base_j = primary["twenty_six_class"]["mean_pairwise_jaccard"] |
| base_cos = primary["distributional_probe_26_class"]["mean_pairwise_cosine_similarity"] |
| base_ge3_cos = primary["well_labelled_distributional_probe_26_class_ge3"]["mean_pairwise_cosine_similarity"] |
| base_prevalence = primary["prevalence_distributional_metric_26_class"] |
| verdict = report["character_verdict"] |
| rows.append(f"| 26-class | 26 | {base_j:.3f} | baseline |") |
| for name, result in primary["groupings"].items(): |
| j = result["discrimination_precheck"]["mean_pairwise_jaccard"] |
| delta = j - base_j |
| rows.append(f"| {name} | {result['n_superclasses']} | {j:.3f} | {delta:+.3f} |") |
| density = primary["label_density_26_class"] |
| density_rows = [ |
| "| 26-class | " |
| f"{density['avg_distinct_classes_per_labelled_formula']:.2f} | " |
| f"{density['avg_labelled_materials_per_labelled_formula']:.2f} |" |
| ] |
| for name, result in primary["groupings"].items(): |
| d = result["label_density"] |
| density_rows.append( |
| f"| {name} | {d['avg_distinct_classes_per_labelled_formula']:.2f} | " |
| f"{d['avg_labelled_materials_per_labelled_formula']:.2f} |" |
| ) |
| superclass_sets = [] |
| for name, result in primary["groupings"].items(): |
| profiles = result["discrimination_precheck"]["genre_profiles"] |
| superclass_sets.append(f"### {name}") |
| for genre, classes in sorted(profiles.items()): |
| superclass_sets.append(f"- `{genre}`: {', '.join(classes)}") |
| superclass_sets.append("") |
| metric_rows = [ |
| "| 26-class | " |
| f"{primary['distributional_probe_26_class']['mean_pairwise_cosine_similarity']:.3f} | " |
| f"{primary['well_labelled_distributional_probe_26_class_ge3']['mean_pairwise_cosine_similarity']:.3f} | " |
| f"{primary['material_level_probe_26_class']['mean_pairwise_cosine_similarity']:.3f} |", |
| ] |
| for name, result in primary["groupings"].items(): |
| metric_rows.append( |
| f"| {name} | " |
| f"{result['distributional_probe']['mean_pairwise_cosine_similarity']:.3f} | " |
| f"{result['well_labelled_distributional_probe_ge3']['mean_pairwise_cosine_similarity']:.3f} | " |
| f"{result['material_level_probe']['mean_pairwise_cosine_similarity']:.3f} |" |
| ) |
| threshold_rows = [] |
| for name, result in primary["groupings"].items(): |
| for row in result["threshold_sensitivity"]: |
| threshold_rows.append(f"| {name} | {row['threshold']:.0%} | {row['mean_pairwise_jaccard']:.3f} |") |
| prevalence_metric_rows = [ |
| "| 26-class | 26 | " |
| f"{base_prevalence['mean_pairwise_cosine_distance']:.3f} | " |
| f"{base_prevalence['mean_pairwise_js_divergence_bits']:.3f} |" |
| ] |
| for name, result in primary["groupings"].items(): |
| metric = result["prevalence_distributional_metric"] |
| prevalence_metric_rows.append( |
| f"| {name} | {result['n_superclasses']} | " |
| f"{metric['mean_pairwise_cosine_distance']:.3f} | " |
| f"{metric['mean_pairwise_js_divergence_bits']:.3f} |" |
| ) |
| distributional_matrices = [ |
| "## Unthresholded Prevalence Distributional Metric", |
| "", |
| "This is the corrected character discrimination test. Vectors retain the raw per-genre prevalence " |
| "of each class before any active-set thresholding; cosine is reported as distance (`1 - similarity`) " |
| "and JS divergence is reported in bits.", |
| "", |
| "| Axis | classes | mean cosine distance | mean JS divergence (bits) |", |
| "| --- | ---: | ---: | ---: |", |
| *prevalence_metric_rows, |
| "", |
| *matrix_markdown( |
| "26-class cosine distance", |
| base_prevalence["cosine_distance_matrix"], |
| ), |
| *matrix_markdown( |
| "26-class JS divergence (bits)", |
| base_prevalence["js_divergence_bits_matrix"], |
| ), |
| ] |
| for name, result in primary["groupings"].items(): |
| metric = result["prevalence_distributional_metric"] |
| distributional_matrices.extend(matrix_markdown(f"{name} cosine distance", metric["cosine_distance_matrix"])) |
| distributional_matrices.extend(matrix_markdown(f"{name} JS divergence (bits)", metric["js_divergence_bits_matrix"])) |
| md = [ |
| "# Character grouping diagnostic v8", |
| "", |
| f"Primary snapshot: `{primary['label_snapshot']}` (`{primary['label_path']}`).", |
| "", |
| "## Verdict", |
| "", |
| f"**Character verdict: {verdict['verdict']}.** Under the corrected 26-class unthresholded " |
| f"prevalence metric, mean cosine distance is {verdict['mean_pairwise_cosine_distance']:.3f} " |
| f"and mean JS divergence is {verdict['mean_pairwise_js_divergence_bits']:.3f} bits. This supports " |
| "weak but genuine distributional separation, suitable only as a secondary axis pending human decision.", |
| "", |
| "**Artifact.** The perfect superclass Jaccard is mechanical: 1.000 = all genres have identical " |
| "thresholded superclass active sets, NOT improved discrimination. After collapse, every genre profile " |
| "contains every available superclass, so set-presence Jaccard has no room to differ. Distributional " |
| f"class-count metrics are not perfect (26-class all-labelled mean cosine {base_cos:.3f}; " |
| f"26-class >=3-label mean cosine {base_ge3_cos:.3f}), so the prior character-is-non-discriminative " |
| "verdict is retracted.", |
| "", |
| "| Axis | classes | mean pairwise Jaccard | delta vs 26 |", |
| "| --- | ---: | ---: | ---: |", |
| *rows, |
| "", |
| "## Superclass Sets Compared", |
| "", |
| *superclass_sets, |
| "## Label Density", |
| "", |
| "| Axis | avg distinct classes / labelled formula | avg labelled materials / labelled formula |", |
| "| --- | ---: | ---: |", |
| *density_rows, |
| "", |
| "## Corrected Distributional Metrics", |
| "", |
| "| Axis | mean cosine, all labelled | mean cosine, >=3 classes | material-level mean cosine |", |
| "| --- | ---: | ---: | ---: |", |
| *metric_rows, |
| "", |
| "Lower cosine means more genre-profile separation. The 26-class axis keeps the strongest separation; " |
| "collapsing to superclasses weakens it and made presence-Jaccard unusable. `best_grouping` is null " |
| "unless a grouped axis materially improves on 26-class; the lowest grouped Jaccard is only recorded " |
| "as `lowest_jaccard_grouping` for audit.", |
| "", |
| *distributional_matrices, |
| "## Threshold Sensitivity", |
| "", |
| "| Grouping | activity threshold | mean pairwise Jaccard |", |
| "| --- | ---: | ---: |", |
| *threshold_rows, |
| "", |
| f"Recommendation: **{report['recommendation']}**", |
| "", |
| f"Character verdict policy: **{verdict['interpretation']}**", |
| "", |
| "Substantivity remains confirmed and publishable independent of this character diagnostic. " |
| "Next human decision: ship substantivity, resume character only under the corrected 26-class " |
| "distributional circuit-breaker, or park character for a trajectory-proportion pivot.", |
| "", |
| "No new sourcing, label expansion, dataset assembly, training, or upload was performed.", |
| ] |
| OUT_MD.write_text("\n".join(md) + "\n", encoding="utf-8") |
|
|
|
|
| def main() -> None: |
| records = census.load_formula_records() |
| snapshots = { |
| name: evaluate_snapshot(name, path, records) |
| for name, path in LABEL_SNAPSHOTS.items() |
| if path.exists() |
| } |
| primary_name = "v8_2_clause_guard_dominance" |
| primary = snapshots[primary_name] |
| report = { |
| "pimt_version": "v8", |
| "diagnostic": "character_grouping_existing_labels_only", |
| "primary_snapshot": primary_name, |
| "primary_snapshot_result": primary, |
| "character_verdict": character_verdict(primary), |
| "all_snapshot_results": snapshots, |
| "recommendation": recommendation(primary), |
| "stop": "NO_NEW_SOURCING_NO_LABEL_EXPANSION_NO_DATASET_ASSEMBLY_NO_TRAINING_NO_HF_UPLOAD", |
| } |
| OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") |
| write_markdown(report) |
| print(json.dumps({ |
| "primary_snapshot": primary_name, |
| "twenty_six_class_jaccard": primary["twenty_six_class"]["mean_pairwise_jaccard"], |
| "best_grouping": primary["best_grouping"], |
| "best_grouping_jaccard": primary["best_grouping_mean_pairwise_jaccard"], |
| "materially_better_than_26_class": primary["materially_better_than_26_class"], |
| "recommendation": report["recommendation"], |
| }, indent=2, sort_keys=True)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
