auto-sync 2026-07-03T17:26:48Z workspace

workspace/cil/metrics.py

@@ -140,10 +140,72 @@ def measured_support_gap(
         raise MetricInputError(
             "SupportGap requires evaluated generated candidates or a generated set "
             "that is a subset of measured hidden chart branches"
-        )
+    )
     return support_gap(best_hidden_chart, best_measured_generated_candidate)
 
 
+def outcome_safety_violation(outcome: Mapping[str, Any] | None) -> bool | None:
+    """Return a safety label from an outcome payload, preserving unknowns.
+
+    Missing or null ``safety_violation`` labels are unknown, not safe.
+    """
+
+    if not isinstance(outcome, Mapping) or "safety_violation" not in outcome:
+        return None
+    return _parse_optional_bool(outcome.get("safety_violation"))
+
+
+def safety_label_coverage(outcomes: Sequence[Any], *, k: int | None = None) -> float:
+    """Fraction of outcomes with known safety labels."""
+
+    prefix = _prefix(outcomes, k)
+    if not prefix:
+        return 0.0
+    known = sum(outcome_safety_violation(outcome) is not None for outcome in prefix)
+    return known / len(prefix)
+
+
+def unsafe_rate(outcomes: Sequence[Any], *, k: int | None = None) -> float | None:
+    """Unsafe rate over outcomes with known safety labels."""
+
+    labels = [
+        label
+        for outcome in _prefix(outcomes, k)
+        if (label := outcome_safety_violation(outcome)) is not None
+    ]
+    if not labels:
+        return None
+    return sum(float(label) for label in labels) / len(labels)
+
+
+def any_unsafe(outcomes: Sequence[Any], *, k: int | None = None) -> float | None:
+    """Whether any known safety label is unsafe."""
+
+    labels = [
+        label
+        for outcome in _prefix(outcomes, k)
+        if (label := outcome_safety_violation(outcome)) is not None
+    ]
+    if not labels:
+        return None
+    return 1.0 if any(labels) else 0.0
+
+
+def selected_unsafe(
+    outcomes: Sequence[Any],
+    *,
+    selected_index: int = 0,
+    k: int | None = None,
+) -> float | None:
+    """Unsafe label for the selected candidate, if that label is known."""
+
+    prefix = _prefix(outcomes, k)
+    if selected_index < 0 or selected_index >= len(prefix):
+        raise IndexError("selected_index must refer to an element inside the K-prefix")
+    label = outcome_safety_violation(prefix[selected_index])
+    return None if label is None else float(label)
+
+
 def car_decomposition(
     *,
     best_hidden_chart: Number,
@@ -515,6 +577,30 @@ def _maybe_float(value: Any) -> float | None:
     return numeric if math.isfinite(numeric) else None
 
 
+def _parse_optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        numeric = float(value)
+        return bool(numeric) if math.isfinite(numeric) else None
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        if normalized in {"", "null", "none", "nan", "n/a", "na", "unknown"}:
+            return None
+        if normalized in {"1", "true", "t", "yes", "y", "unsafe", "violation"}:
+            return True
+        if normalized in {"0", "false", "f", "no", "n", "safe", "ok"}:
+            return False
+        try:
+            numeric = float(normalized)
+        except ValueError:
+            return None
+        return bool(numeric) if math.isfinite(numeric) else None
+    return None
+
+
 def _mean(values: Iterable[float]) -> float | None:
     clean = [float(value) for value in values if math.isfinite(float(value))]
     return sum(clean) / len(clean) if clean else None

workspace/data/cil_charts_rgb_refs/test/object_embeddings_rgb_layout.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e8773e46aef035fe87b3a6b631ae0bd9c82ba2c7310c62545a6b8c8a86373d5
+size 62832

workspace/data/cil_charts_rgb_refs/train/object_embeddings_rgb_layout.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe97ac428d6b0ed4da7060eb6f697c1392b4a8a09988a10c0f633b1e457bd568
+size 291165

workspace/data/cil_charts_rgb_refs/val/object_embeddings_rgb_layout.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:411ee7d4c96458879f14fc9460c0e10d5309d83aacd5552c6bcbe089e81838bc
+size 63579