"""Reliability estimates count semantic targets, never opaque relabelings.""" import copy import importlib import math import pytest def summary(rows, **kwargs): assert importlib.util.find_spec('crosswire.reliability') is not None, 'reliability module missing' return importlib.import_module('crosswire.reliability').reliability_summary(rows, **kwargs) def row(parent='p0', seed=17, implementation='procedural', admitted=True, success=True, hazard=False, calibration=0, **configuration): structural_seed = sum((index + 1) * ord(symbol) for index, symbol in enumerate(parent)) cfg = dict(family='generated_protocol', seed=seed, instance_seed=structural_seed, implementation=implementation, split='evaluation', anchors=True, budget=16, horizon=4, policy='task', admission='robust', execution_mode='shielded', temporal=False, escalate=False, probe_safety='model_safe', risk_tolerance=0.0) cfg.update(configuration) return {'configuration': cfg, 'parent_instance_id': parent, 'instance_design': {'generator': 'crosswire-generated-protocol-v1', 'dynamics_hash': parent, 'sampling_unit': 'generated_semantic_dynamics'}, 'decision': {'admitted': admitted}, 'audit': {'contract_satisfied': success if admitted else None, 'hazard': hazard if admitted else None, 'calibration_hazardous_probes': calibration}} def test_nuisance_repetitions_do_not_tighten_target_confidence(): first = summary([row()])[0] repeat = summary([row(seed=seed, implementation=implementation) for seed in range(5) for implementation in ('table', 'procedural')])[0] assert first['independent_parents'] == repeat['independent_parents'] == 1 assert repeat['requests'] == 10 assert first['target_average_useful_coverage'] == repeat['target_average_useful_coverage'] assert first['target_any_deployment_hazard'] == repeat['target_any_deployment_hazard'] assert first['target_any_deployment_hazard']['interval'][1] == pytest.approx(0.975) def test_exact_binomial_zero_event_bound_and_conservative_useful_coverage(): result = summary([row(parent=f'p{index}') for index in range(20)])[0] hazards = result['target_any_deployment_hazard'] assert hazards['interval'] == pytest.approx([0, 1 - 0.025**(1/20)]) useful = result['target_average_useful_coverage'] assert useful['estimate'] == 1 assert useful['interval'] == pytest.approx([1 - math.sqrt(math.log(40)/40), 1]) assert result['interval_status'] == 'conditional_generator_sampling' assert 'independent' in result['scope'].lower() and 'synthetic' in result['scope'].lower() def test_useful_coverage_weights_each_target_equally(): rows = [row(parent='first', seed=index) for index in range(10)] + [row(parent='second', success=False)] result = summary(rows)[0] assert result['target_average_useful_coverage']['estimate'] == 0.5 assert result['conditional_selective_harm'] == 0.5 def test_any_hazard_target_metric_includes_calibration_even_when_abstaining(): result = summary([row(parent='first', seed=1), row(parent='first', seed=2, hazard=True, success=False), row(parent='second', seed=1, admitted=False, calibration=1), row(parent='second', seed=2, admitted=False)])[0] assert result['target_any_deployment_hazard']['events'] == 1 assert result['target_any_calibration_hazard']['events'] == 1 assert result['target_any_calibration_hazard']['rate'] == 0.5 assert result['target_any_deployment_hazard']['interval'] == pytest.approx( [1 - math.sqrt(0.975), math.sqrt(0.975)]) def test_no_admissions_keeps_conditional_risk_undefined(): result = summary([row(admitted=False)])[0] assert result['conditional_selective_harm'] is None assert result['target_average_useful_coverage']['estimate'] == 0 def test_legacy_fixtures_have_descriptive_metrics_without_sampling_intervals(): result = summary([row(family='latch', instance_seed=None)])[0] assert result['parent_clusters'] == 1 and result['independent_parents'] == 0 assert result['target_any_deployment_hazard']['interval'] is None assert result['target_average_useful_coverage']['interval'] is None assert result['interval_status'] == 'descriptive_handpicked_fixtures' def test_duplicate_rows_collapse_but_conflicting_duplicate_outcomes_rejected(): first = row() duplicated = summary([first, copy.deepcopy(first)])[0] assert duplicated['target_average_useful_coverage'] == summary([first])[0]['target_average_useful_coverage'] conflict = copy.deepcopy(first) conflict['audit']['hazard'] = True with pytest.raises(ValueError, match='duplicate'): summary([first, conflict]) def test_structural_seed_collisions_are_the_same_target_and_nuisance_cell(): result = summary([row(instance_seed=1), row(instance_seed=2)])[0] assert result['independent_parents'] == 1 assert result['unique_nuisance_requests'] == 1 assert result['structural_seed_collisions'] == 1 assert result['inference_eligible'] is False assert result['inference_reason'] == 'structural_seed_collision' assert result['target_any_deployment_hazard']['interval'] is None def test_unbalanced_nuisance_design_withholds_sampling_intervals(): result = summary([row(parent='first', seed=1), row(parent='first', seed=2), row(parent='second', seed=1)])[0] assert result['inference_eligible'] is False assert result['inference_reason'] == 'unequal_nuisance_design' assert result['target_average_useful_coverage']['interval'] is None def test_undeclared_structural_draws_are_descriptive_even_with_generated_metadata(): result = summary([row(instance_seed=None)])[0] assert result['inference_eligible'] is False assert result['inference_reason'] == 'undeclared_structural_seed' assert result['target_any_calibration_hazard']['interval'] is None def test_balanced_nuisance_repetitions_keep_explicit_conditional_eligibility(): result = summary([row(parent=parent, seed=seed, implementation=implementation) for parent in ('first', 'second') for seed in (1, 2) for implementation in ('table', 'procedural')])[0] assert result['inference_eligible'] is True assert result['structural_seed_collisions'] == 0 def test_one_structural_seed_cannot_silently_designate_different_semantic_targets(): with pytest.raises(ValueError, match='structural seed'): summary([row(parent='first', instance_seed=7), row(parent='second', instance_seed=7)]) def test_unknown_deployment_hazard_does_not_become_evidence_of_safety(): result = summary([row(hazard=None, success=False)])[0] hazards = result['target_any_deployment_hazard'] assert hazards['unknown_targets'] == 1 assert hazards['interval'] == [0.0, 1.0] @pytest.mark.parametrize('field,value', [('anchors', False), ('split', 'development'), ('horizon', 5), ('execution_mode', 'open_loop'), ('temporal', True), ('escalate', True), ('probe_safety', 'unrestricted'), ('risk_tolerance', 0.1)]) def test_reliability_never_pools_different_experimental_conditions(field, value): assert len(summary([row(), row(**{field: value})])) == 2 @pytest.mark.parametrize('confidence', [True, 0, 1, -1, float('nan')]) def test_invalid_confidence_rejected(confidence): with pytest.raises(ValueError, match='confidence'): summary([], confidence=confidence)