talkingheadbench / tests /smoke /test_node2_boundaries.py
elix3r's picture
Upload folder using huggingface_hub
ab34aa7 verified
Raw
History Blame Contribute Delete
18.7 kB
"""
Smoke tests: Node 2 (Parameter Anomaly Detector) boundary conditions.
Covers:
1. Empty / unknown config keys produce safe output.
2. Each rule fires only for its declared regime/context (and on the
correct side of each threshold).
3. Risk-level escalation logic (safe β†’ marginal β†’ risky β†’ dangerous).
4. Predicted failure-mode deduplication.
5. Directional-fix count mirrors anomaly count.
All rules are verified at exact boundary values so we confirm ``<`` vs
``<=`` strictness throughout.
"""
from __future__ import annotations
from src.envs.subenv1.node2_param_anomaly import detect_param_anomalies
from src.schemas.subenv1 import ParamAnomalyAction, ParamAnomalyObservation
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
_DEFAULTS: dict = dict(
proposed_config={},
regime="frontal_simple",
identified_risk_factors=[],
image_usability_score=0.7,
face_occupancy_ratio=0.6,
estimated_yaw_degrees=5.0,
background_complexity_score=0.3,
lighting_uniformity_score=0.7,
occlusion_detected=False,
prompt_identity_anchoring=0.7,
prompt_token_count=40,
conflicting_descriptors=[],
)
def base_obs(**overrides) -> ParamAnomalyObservation:
"""Return a ParamAnomalyObservation in the safe zone with applied overrides.
Defaults keep every rule dormant:
- proposed_config empty β†’ no parameter evaluated
- regime "frontal_simple" β†’ denoise_alt rule cannot fire
- background_complexity_score 0.3 ≀ 0.6 β†’ cfg rule cannot fire
- image_usability_score 0.7 β‰₯ 0.5 β†’ eta rule cannot fire
"""
return ParamAnomalyObservation(**{**_DEFAULTS, **overrides})
def run(obs: ParamAnomalyObservation) -> ParamAnomalyAction:
return detect_param_anomalies(obs)
# ===========================================================================
# 1. Empty / unknown config keys
# ===========================================================================
class TestEmptyAndUnknownConfig:
"""Only keys present in proposed_config are evaluated β€” unrecognised keys
must be silently ignored, not raise or produce spurious anomalies."""
def test_empty_config_produces_safe(self):
"""Empty proposed_config β†’ safe level, no anomalies."""
action = run(base_obs())
assert action.config_risk_level == "safe"
assert action.anomalies == []
def test_unknown_param_keys_ignored(self):
"""Unrecognised parameter names produce no anomalies."""
action = run(base_obs(proposed_config={"nonexistent_param": 99.9}))
assert action.config_risk_level == "safe"
assert action.anomalies == []
def test_known_key_absent_is_safe_even_with_triggering_context(self):
"""If 'denoise_alt' is absent from proposed_config, the rule never runs
even when the regime is 'non_frontal'."""
obs = base_obs(regime="non_frontal", proposed_config={})
action = run(obs)
assert action.anomalies == []
# ===========================================================================
# 2. denoise_alt rule
# ===========================================================================
class TestDenoiseAltRule:
"""Rule: regime=='non_frontal' and val < 0.45 β†’ severe / reference_token_dropout
regime=='complex_background' and val > 0.65 β†’ moderate / identity_collapse
"""
# --- regime guard ---
def test_denoise_alt_severe_only_for_non_frontal(self):
"""val=0.25 (< 0.45) with regime='frontal_simple' β†’ no anomaly."""
obs = base_obs(
regime="frontal_simple",
proposed_config={"denoise_alt": 0.25},
)
action = run(obs)
assert not any(a.severity == "severe" for a in action.anomalies)
def test_denoise_alt_severe_fires_for_non_frontal(self):
"""val=0.25 (< 0.45) with regime='non_frontal' β†’ severe anomaly."""
obs = base_obs(
regime="non_frontal",
proposed_config={"denoise_alt": 0.25},
)
action = run(obs)
assert any(
a.severity == "severe" and a.parameter == "denoise_alt"
for a in action.anomalies
)
def test_denoise_alt_non_frontal_linked_failure_mode(self):
"""Severe denoise_alt anomaly links to reference_token_dropout."""
obs = base_obs(
regime="non_frontal",
proposed_config={"denoise_alt": 0.25},
)
action = run(obs)
assert any(
a.linked_failure_mode == "reference_token_dropout"
for a in action.anomalies
)
# --- exact boundary: val < 0.45 (strict) ---
def test_denoise_alt_exact_boundary_045_no_anomaly(self):
"""val == 0.45 is NOT < 0.45 β†’ no anomaly for non_frontal."""
obs = base_obs(
regime="non_frontal",
proposed_config={"denoise_alt": 0.45},
)
action = run(obs)
assert action.anomalies == []
def test_denoise_alt_exact_boundary_0449_anomaly(self):
"""val == 0.449 < 0.45 β†’ severe anomaly for non_frontal."""
obs = base_obs(
regime="non_frontal",
proposed_config={"denoise_alt": 0.449},
)
action = run(obs)
assert any(
a.severity == "severe" and a.parameter == "denoise_alt"
for a in action.anomalies
)
# --- complex_background branch ---
def test_denoise_alt_complex_background_high_value(self):
"""val=0.70 > 0.65 with regime='complex_background' β†’ moderate anomaly."""
obs = base_obs(
regime="complex_background",
proposed_config={"denoise_alt": 0.70},
)
action = run(obs)
assert any(
a.severity == "moderate" and a.parameter == "denoise_alt"
for a in action.anomalies
)
def test_denoise_alt_complex_background_exact_boundary_065_no_anomaly(self):
"""val == 0.65 is NOT > 0.65 β†’ no anomaly for complex_background."""
obs = base_obs(
regime="complex_background",
proposed_config={"denoise_alt": 0.65},
)
action = run(obs)
assert action.anomalies == []
def test_denoise_alt_complex_background_just_above_065(self):
"""val == 0.651 > 0.65 β†’ moderate anomaly for complex_background."""
obs = base_obs(
regime="complex_background",
proposed_config={"denoise_alt": 0.651},
)
action = run(obs)
assert any(
a.severity == "moderate" and a.parameter == "denoise_alt"
for a in action.anomalies
)
# ===========================================================================
# 3. cfg rule
# ===========================================================================
class TestCfgRule:
"""Rule: background_complexity_score > 0.6 AND cfg_val > 7.0 β†’ moderate / background_bleed."""
def test_cfg_only_fires_with_complex_background_above_06(self):
"""bg_complexity == 0.59 ≀ 0.6 β†’ cfg rule cannot fire even with high cfg."""
obs = base_obs(
background_complexity_score=0.59,
proposed_config={"cfg": 8.0},
)
action = run(obs)
assert action.anomalies == []
def test_cfg_fires_when_bg_above_06(self):
"""bg_complexity == 0.61 > 0.6 and cfg == 8.0 > 7.0 β†’ moderate anomaly."""
obs = base_obs(
background_complexity_score=0.61,
proposed_config={"cfg": 8.0},
)
action = run(obs)
assert any(
a.severity == "moderate" and a.parameter == "cfg"
for a in action.anomalies
)
def test_cfg_exact_bg_boundary_06_no_anomaly(self):
"""bg_complexity == 0.6 is NOT > 0.6 β†’ no anomaly."""
obs = base_obs(
background_complexity_score=0.6,
proposed_config={"cfg": 8.0},
)
action = run(obs)
assert action.anomalies == []
def test_cfg_exact_val_boundary_70_no_anomaly(self):
"""cfg == 7.0 is NOT > 7.0 β†’ no anomaly (bg_complexity safely above 0.6)."""
obs = base_obs(
background_complexity_score=0.7,
proposed_config={"cfg": 7.0},
)
action = run(obs)
assert action.anomalies == []
def test_cfg_just_above_70_fires(self):
"""cfg == 7.01 > 7.0 with bg_complexity=0.7 > 0.6 β†’ anomaly."""
obs = base_obs(
background_complexity_score=0.7,
proposed_config={"cfg": 7.01},
)
action = run(obs)
assert any(a.parameter == "cfg" for a in action.anomalies)
def test_cfg_linked_failure_mode_is_background_bleed(self):
"""cfg anomaly always links to background_bleed."""
obs = base_obs(
background_complexity_score=0.7,
proposed_config={"cfg": 8.0},
)
action = run(obs)
assert any(
a.linked_failure_mode == "background_bleed" for a in action.anomalies
)
# ===========================================================================
# 4. eta rule
# ===========================================================================
class TestEtaRule:
"""Rule: eta_val > 0.12 AND image_usability_score < 0.5 β†’ moderate / identity_collapse."""
def test_eta_only_fires_with_weak_reference_usability_below_05(self):
"""usability == 0.5 is NOT < 0.5 β†’ eta rule cannot fire."""
obs = base_obs(
image_usability_score=0.5,
proposed_config={"eta": 0.13},
)
action = run(obs)
assert action.anomalies == []
def test_eta_fires_when_usability_below_05(self):
"""usability == 0.49 < 0.5 and eta == 0.13 > 0.12 β†’ moderate anomaly."""
obs = base_obs(
image_usability_score=0.49,
proposed_config={"eta": 0.13},
)
action = run(obs)
assert any(
a.severity == "moderate" and a.parameter == "eta"
for a in action.anomalies
)
def test_eta_exact_val_boundary_012_no_anomaly(self):
"""eta == 0.12 is NOT > 0.12 β†’ no anomaly (usability safely below 0.5)."""
obs = base_obs(
image_usability_score=0.4,
proposed_config={"eta": 0.12},
)
action = run(obs)
assert action.anomalies == []
def test_eta_just_above_012_fires(self):
"""eta == 0.121 > 0.12 with usability=0.4 < 0.5 β†’ anomaly."""
obs = base_obs(
image_usability_score=0.4,
proposed_config={"eta": 0.121},
)
action = run(obs)
assert any(a.parameter == "eta" for a in action.anomalies)
def test_eta_linked_failure_mode_is_identity_collapse(self):
"""eta anomaly always links to identity_collapse."""
obs = base_obs(
image_usability_score=0.4,
proposed_config={"eta": 0.13},
)
action = run(obs)
assert any(
a.linked_failure_mode == "identity_collapse" for a in action.anomalies
)
# ===========================================================================
# 5. Config risk level escalation
# ===========================================================================
class TestRiskLevelEscalation:
"""safe β†’ marginal β†’ risky β†’ dangerous strictly via severity counts."""
def test_risk_level_safe_no_anomalies(self):
"""No anomalies β†’ safe."""
action = run(base_obs())
assert action.config_risk_level == "safe"
def test_risk_level_marginal_one_moderate(self):
"""One moderate anomaly β†’ marginal (not risky)."""
# cfg moderate: bg_complexity=0.7, cfg=8.0
obs = base_obs(
background_complexity_score=0.7,
proposed_config={"cfg": 8.0},
)
action = run(obs)
assert action.config_risk_level == "marginal"
def test_risk_level_risky_requires_two_moderate(self):
"""Two moderate anomalies β†’ risky.
cfg (background_bleed) + eta (identity_collapse):
bg_complexity=0.7 > 0.6, cfg=8.0 > 7.0
usability=0.4 < 0.5, eta=0.13 > 0.12
"""
obs = base_obs(
background_complexity_score=0.7,
image_usability_score=0.4,
proposed_config={"cfg": 8.0, "eta": 0.13},
)
action = run(obs)
assert action.config_risk_level == "risky"
def test_risk_level_dangerous_requires_severe(self):
"""One severe anomaly β†’ dangerous, regardless of moderate count."""
obs = base_obs(
regime="non_frontal",
proposed_config={"denoise_alt": 0.25},
)
action = run(obs)
assert action.config_risk_level == "dangerous"
def test_risk_level_dangerous_overrides_moderate(self):
"""Severe + moderate simultaneously β†’ still dangerous (not risky)."""
obs = base_obs(
regime="non_frontal",
background_complexity_score=0.7,
proposed_config={"denoise_alt": 0.25, "cfg": 8.0},
)
action = run(obs)
assert action.config_risk_level == "dangerous"
def test_only_moderate_anomalies_never_dangerous(self):
"""Two moderate anomalies β†’ risky, never dangerous."""
obs = base_obs(
background_complexity_score=0.7,
image_usability_score=0.4,
proposed_config={"cfg": 8.0, "eta": 0.13},
)
action = run(obs)
assert action.config_risk_level != "dangerous"
# ===========================================================================
# 6. Failure-mode deduplication
# ===========================================================================
class TestFailureModeDeduplicated:
"""predicted_failure_modes must contain each mode at most once."""
def test_predicted_failure_modes_deduplicated(self):
"""Two anomalies with the same linked_failure_mode='identity_collapse':
denoise_alt in complex_background (val=0.70 > 0.65) + eta (val=0.13,
usability=0.4 < 0.5). Only one 'identity_collapse' entry expected.
"""
obs = base_obs(
regime="complex_background",
image_usability_score=0.4,
proposed_config={"denoise_alt": 0.70, "eta": 0.13},
)
action = run(obs)
# Both anomalies link to identity_collapse
assert len(action.anomalies) == 2
assert action.predicted_failure_modes.count("identity_collapse") == 1
def test_predicted_failure_modes_order_is_insertion_order(self):
"""Failure modes appear in the order the anomalies were detected.
denoise_alt is checked before eta in the implementation, so
reference_token_dropout should come before identity_collapse.
"""
obs = base_obs(
regime="non_frontal",
image_usability_score=0.4,
proposed_config={"denoise_alt": 0.25, "eta": 0.13},
)
action = run(obs)
modes = action.predicted_failure_modes
assert modes.index("reference_token_dropout") < modes.index("identity_collapse")
def test_multiple_distinct_failure_modes_all_present(self):
"""Three anomalies with three distinct failure modes β†’ three entries."""
# reference_token_dropout (denoise_alt/non_frontal)
# background_bleed (cfg)
# identity_collapse (eta)
obs = base_obs(
regime="non_frontal",
background_complexity_score=0.7,
image_usability_score=0.4,
proposed_config={"denoise_alt": 0.25, "cfg": 8.0, "eta": 0.13},
)
action = run(obs)
assert len(action.predicted_failure_modes) == 3
assert set(action.predicted_failure_modes) == {
"reference_token_dropout",
"background_bleed",
"identity_collapse",
}
# ===========================================================================
# 7. Directional fix count
# ===========================================================================
class TestDirectionalFixes:
"""One directional fix is generated per anomaly for every known rule branch."""
def test_no_fixes_for_safe_config(self):
"""No anomalies β†’ empty directional_fixes list."""
action = run(base_obs())
assert action.directional_fixes == []
def test_directional_fixes_one_per_anomaly(self):
"""Three anomalies (reference_token_dropout + background_bleed +
identity_collapse/eta) β†’ three directional fixes.
Uses non_frontal to trigger denoise_alt severe + cfg moderate +
eta moderate. background_complexity_score > 0.6 for cfg;
image_usability_score < 0.5 for eta.
"""
obs = base_obs(
regime="non_frontal",
background_complexity_score=0.7,
image_usability_score=0.4,
proposed_config={"denoise_alt": 0.25, "cfg": 8.0, "eta": 0.13},
)
action = run(obs)
assert len(action.anomalies) == 3
assert len(action.directional_fixes) == 3
def test_fix_for_reference_token_dropout_increases(self):
"""reference_token_dropout β†’ fix direction is 'increase'."""
obs = base_obs(
regime="non_frontal",
proposed_config={"denoise_alt": 0.25},
)
action = run(obs)
td_fixes = [f for f in action.directional_fixes if f.target == "reference_token_strength"]
assert len(td_fixes) == 1
assert td_fixes[0].direction == "increase"
def test_fix_for_background_bleed_decreases(self):
"""background_bleed β†’ fix direction is 'decrease' on guidance_scale."""
obs = base_obs(
background_complexity_score=0.7,
proposed_config={"cfg": 8.0},
)
action = run(obs)
bg_fixes = [f for f in action.directional_fixes if f.target == "guidance_scale"]
assert len(bg_fixes) == 1
assert bg_fixes[0].direction == "decrease"
def test_fix_for_eta_identity_collapse_decreases(self):
"""identity_collapse via eta β†’ fix direction is 'decrease' on stochasticity."""
obs = base_obs(
image_usability_score=0.4,
proposed_config={"eta": 0.13},
)
action = run(obs)
eta_fixes = [f for f in action.directional_fixes if "eta" in f.target]
assert len(eta_fixes) == 1
assert eta_fixes[0].direction == "decrease"