Spaces:
Running
Running
| """Reliability estimates count semantic targets, never opaque relabelings.""" | |
| import copy | |
| import importlib | |
| import math | |
| import pytest | |
| def summary(rows, **kwargs): | |
| assert importlib.util.find_spec('crosswire.reliability') is not None, 'reliability module missing' | |
| return importlib.import_module('crosswire.reliability').reliability_summary(rows, **kwargs) | |
| def row(parent='p0', seed=17, implementation='procedural', admitted=True, | |
| success=True, hazard=False, calibration=0, **configuration): | |
| structural_seed = sum((index + 1) * ord(symbol) for index, symbol in enumerate(parent)) | |
| cfg = dict(family='generated_protocol', seed=seed, instance_seed=structural_seed, implementation=implementation, | |
| split='evaluation', anchors=True, budget=16, horizon=4, policy='task', admission='robust', | |
| execution_mode='shielded', temporal=False, escalate=False, | |
| probe_safety='model_safe', risk_tolerance=0.0) | |
| cfg.update(configuration) | |
| return {'configuration': cfg, 'parent_instance_id': parent, | |
| 'instance_design': {'generator': 'crosswire-generated-protocol-v1', | |
| 'dynamics_hash': parent, 'sampling_unit': 'generated_semantic_dynamics'}, | |
| 'decision': {'admitted': admitted}, | |
| 'audit': {'contract_satisfied': success if admitted else None, | |
| 'hazard': hazard if admitted else None, | |
| 'calibration_hazardous_probes': calibration}} | |
| def test_nuisance_repetitions_do_not_tighten_target_confidence(): | |
| first = summary([row()])[0] | |
| repeat = summary([row(seed=seed, implementation=implementation) | |
| for seed in range(5) for implementation in ('table', 'procedural')])[0] | |
| assert first['independent_parents'] == repeat['independent_parents'] == 1 | |
| assert repeat['requests'] == 10 | |
| assert first['target_average_useful_coverage'] == repeat['target_average_useful_coverage'] | |
| assert first['target_any_deployment_hazard'] == repeat['target_any_deployment_hazard'] | |
| assert first['target_any_deployment_hazard']['interval'][1] == pytest.approx(0.975) | |
| def test_exact_binomial_zero_event_bound_and_conservative_useful_coverage(): | |
| result = summary([row(parent=f'p{index}') for index in range(20)])[0] | |
| hazards = result['target_any_deployment_hazard'] | |
| assert hazards['interval'] == pytest.approx([0, 1 - 0.025**(1/20)]) | |
| useful = result['target_average_useful_coverage'] | |
| assert useful['estimate'] == 1 | |
| assert useful['interval'] == pytest.approx([1 - math.sqrt(math.log(40)/40), 1]) | |
| assert result['interval_status'] == 'conditional_generator_sampling' | |
| assert 'independent' in result['scope'].lower() and 'synthetic' in result['scope'].lower() | |
| def test_useful_coverage_weights_each_target_equally(): | |
| rows = [row(parent='first', seed=index) for index in range(10)] + [row(parent='second', success=False)] | |
| result = summary(rows)[0] | |
| assert result['target_average_useful_coverage']['estimate'] == 0.5 | |
| assert result['conditional_selective_harm'] == 0.5 | |
| def test_any_hazard_target_metric_includes_calibration_even_when_abstaining(): | |
| result = summary([row(parent='first', seed=1), row(parent='first', seed=2, hazard=True, success=False), | |
| row(parent='second', seed=1, admitted=False, calibration=1), | |
| row(parent='second', seed=2, admitted=False)])[0] | |
| assert result['target_any_deployment_hazard']['events'] == 1 | |
| assert result['target_any_calibration_hazard']['events'] == 1 | |
| assert result['target_any_calibration_hazard']['rate'] == 0.5 | |
| assert result['target_any_deployment_hazard']['interval'] == pytest.approx( | |
| [1 - math.sqrt(0.975), math.sqrt(0.975)]) | |
| def test_no_admissions_keeps_conditional_risk_undefined(): | |
| result = summary([row(admitted=False)])[0] | |
| assert result['conditional_selective_harm'] is None | |
| assert result['target_average_useful_coverage']['estimate'] == 0 | |
| def test_legacy_fixtures_have_descriptive_metrics_without_sampling_intervals(): | |
| result = summary([row(family='latch', instance_seed=None)])[0] | |
| assert result['parent_clusters'] == 1 and result['independent_parents'] == 0 | |
| assert result['target_any_deployment_hazard']['interval'] is None | |
| assert result['target_average_useful_coverage']['interval'] is None | |
| assert result['interval_status'] == 'descriptive_handpicked_fixtures' | |
| def test_duplicate_rows_collapse_but_conflicting_duplicate_outcomes_rejected(): | |
| first = row() | |
| duplicated = summary([first, copy.deepcopy(first)])[0] | |
| assert duplicated['target_average_useful_coverage'] == summary([first])[0]['target_average_useful_coverage'] | |
| conflict = copy.deepcopy(first) | |
| conflict['audit']['hazard'] = True | |
| with pytest.raises(ValueError, match='duplicate'): | |
| summary([first, conflict]) | |
| def test_structural_seed_collisions_are_the_same_target_and_nuisance_cell(): | |
| result = summary([row(instance_seed=1), row(instance_seed=2)])[0] | |
| assert result['independent_parents'] == 1 | |
| assert result['unique_nuisance_requests'] == 1 | |
| assert result['structural_seed_collisions'] == 1 | |
| assert result['inference_eligible'] is False | |
| assert result['inference_reason'] == 'structural_seed_collision' | |
| assert result['target_any_deployment_hazard']['interval'] is None | |
| def test_unbalanced_nuisance_design_withholds_sampling_intervals(): | |
| result = summary([row(parent='first', seed=1), row(parent='first', seed=2), | |
| row(parent='second', seed=1)])[0] | |
| assert result['inference_eligible'] is False | |
| assert result['inference_reason'] == 'unequal_nuisance_design' | |
| assert result['target_average_useful_coverage']['interval'] is None | |
| def test_undeclared_structural_draws_are_descriptive_even_with_generated_metadata(): | |
| result = summary([row(instance_seed=None)])[0] | |
| assert result['inference_eligible'] is False | |
| assert result['inference_reason'] == 'undeclared_structural_seed' | |
| assert result['target_any_calibration_hazard']['interval'] is None | |
| def test_balanced_nuisance_repetitions_keep_explicit_conditional_eligibility(): | |
| result = summary([row(parent=parent, seed=seed, implementation=implementation) | |
| for parent in ('first', 'second') for seed in (1, 2) | |
| for implementation in ('table', 'procedural')])[0] | |
| assert result['inference_eligible'] is True | |
| assert result['structural_seed_collisions'] == 0 | |
| def test_one_structural_seed_cannot_silently_designate_different_semantic_targets(): | |
| with pytest.raises(ValueError, match='structural seed'): | |
| summary([row(parent='first', instance_seed=7), row(parent='second', instance_seed=7)]) | |
| def test_unknown_deployment_hazard_does_not_become_evidence_of_safety(): | |
| result = summary([row(hazard=None, success=False)])[0] | |
| hazards = result['target_any_deployment_hazard'] | |
| assert hazards['unknown_targets'] == 1 | |
| assert hazards['interval'] == [0.0, 1.0] | |
| def test_reliability_never_pools_different_experimental_conditions(field, value): | |
| assert len(summary([row(), row(**{field: value})])) == 2 | |
| def test_invalid_confidence_rejected(confidence): | |
| with pytest.raises(ValueError, match='confidence'): | |
| summary([], confidence=confidence) | |