File size: 7,880 Bytes
450ac2f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | """Reliability estimates count semantic targets, never opaque relabelings."""
import copy
import importlib
import math
import pytest
def summary(rows, **kwargs):
assert importlib.util.find_spec('crosswire.reliability') is not None, 'reliability module missing'
return importlib.import_module('crosswire.reliability').reliability_summary(rows, **kwargs)
def row(parent='p0', seed=17, implementation='procedural', admitted=True,
success=True, hazard=False, calibration=0, **configuration):
structural_seed = sum((index + 1) * ord(symbol) for index, symbol in enumerate(parent))
cfg = dict(family='generated_protocol', seed=seed, instance_seed=structural_seed, implementation=implementation,
split='evaluation', anchors=True, budget=16, horizon=4, policy='task', admission='robust',
execution_mode='shielded', temporal=False, escalate=False,
probe_safety='model_safe', risk_tolerance=0.0)
cfg.update(configuration)
return {'configuration': cfg, 'parent_instance_id': parent,
'instance_design': {'generator': 'crosswire-generated-protocol-v1',
'dynamics_hash': parent, 'sampling_unit': 'generated_semantic_dynamics'},
'decision': {'admitted': admitted},
'audit': {'contract_satisfied': success if admitted else None,
'hazard': hazard if admitted else None,
'calibration_hazardous_probes': calibration}}
def test_nuisance_repetitions_do_not_tighten_target_confidence():
first = summary([row()])[0]
repeat = summary([row(seed=seed, implementation=implementation)
for seed in range(5) for implementation in ('table', 'procedural')])[0]
assert first['independent_parents'] == repeat['independent_parents'] == 1
assert repeat['requests'] == 10
assert first['target_average_useful_coverage'] == repeat['target_average_useful_coverage']
assert first['target_any_deployment_hazard'] == repeat['target_any_deployment_hazard']
assert first['target_any_deployment_hazard']['interval'][1] == pytest.approx(0.975)
def test_exact_binomial_zero_event_bound_and_conservative_useful_coverage():
result = summary([row(parent=f'p{index}') for index in range(20)])[0]
hazards = result['target_any_deployment_hazard']
assert hazards['interval'] == pytest.approx([0, 1 - 0.025**(1/20)])
useful = result['target_average_useful_coverage']
assert useful['estimate'] == 1
assert useful['interval'] == pytest.approx([1 - math.sqrt(math.log(40)/40), 1])
assert result['interval_status'] == 'conditional_generator_sampling'
assert 'independent' in result['scope'].lower() and 'synthetic' in result['scope'].lower()
def test_useful_coverage_weights_each_target_equally():
rows = [row(parent='first', seed=index) for index in range(10)] + [row(parent='second', success=False)]
result = summary(rows)[0]
assert result['target_average_useful_coverage']['estimate'] == 0.5
assert result['conditional_selective_harm'] == 0.5
def test_any_hazard_target_metric_includes_calibration_even_when_abstaining():
result = summary([row(parent='first', seed=1), row(parent='first', seed=2, hazard=True, success=False),
row(parent='second', seed=1, admitted=False, calibration=1),
row(parent='second', seed=2, admitted=False)])[0]
assert result['target_any_deployment_hazard']['events'] == 1
assert result['target_any_calibration_hazard']['events'] == 1
assert result['target_any_calibration_hazard']['rate'] == 0.5
assert result['target_any_deployment_hazard']['interval'] == pytest.approx(
[1 - math.sqrt(0.975), math.sqrt(0.975)])
def test_no_admissions_keeps_conditional_risk_undefined():
result = summary([row(admitted=False)])[0]
assert result['conditional_selective_harm'] is None
assert result['target_average_useful_coverage']['estimate'] == 0
def test_legacy_fixtures_have_descriptive_metrics_without_sampling_intervals():
result = summary([row(family='latch', instance_seed=None)])[0]
assert result['parent_clusters'] == 1 and result['independent_parents'] == 0
assert result['target_any_deployment_hazard']['interval'] is None
assert result['target_average_useful_coverage']['interval'] is None
assert result['interval_status'] == 'descriptive_handpicked_fixtures'
def test_duplicate_rows_collapse_but_conflicting_duplicate_outcomes_rejected():
first = row()
duplicated = summary([first, copy.deepcopy(first)])[0]
assert duplicated['target_average_useful_coverage'] == summary([first])[0]['target_average_useful_coverage']
conflict = copy.deepcopy(first)
conflict['audit']['hazard'] = True
with pytest.raises(ValueError, match='duplicate'):
summary([first, conflict])
def test_structural_seed_collisions_are_the_same_target_and_nuisance_cell():
result = summary([row(instance_seed=1), row(instance_seed=2)])[0]
assert result['independent_parents'] == 1
assert result['unique_nuisance_requests'] == 1
assert result['structural_seed_collisions'] == 1
assert result['inference_eligible'] is False
assert result['inference_reason'] == 'structural_seed_collision'
assert result['target_any_deployment_hazard']['interval'] is None
def test_unbalanced_nuisance_design_withholds_sampling_intervals():
result = summary([row(parent='first', seed=1), row(parent='first', seed=2),
row(parent='second', seed=1)])[0]
assert result['inference_eligible'] is False
assert result['inference_reason'] == 'unequal_nuisance_design'
assert result['target_average_useful_coverage']['interval'] is None
def test_undeclared_structural_draws_are_descriptive_even_with_generated_metadata():
result = summary([row(instance_seed=None)])[0]
assert result['inference_eligible'] is False
assert result['inference_reason'] == 'undeclared_structural_seed'
assert result['target_any_calibration_hazard']['interval'] is None
def test_balanced_nuisance_repetitions_keep_explicit_conditional_eligibility():
result = summary([row(parent=parent, seed=seed, implementation=implementation)
for parent in ('first', 'second') for seed in (1, 2)
for implementation in ('table', 'procedural')])[0]
assert result['inference_eligible'] is True
assert result['structural_seed_collisions'] == 0
def test_one_structural_seed_cannot_silently_designate_different_semantic_targets():
with pytest.raises(ValueError, match='structural seed'):
summary([row(parent='first', instance_seed=7), row(parent='second', instance_seed=7)])
def test_unknown_deployment_hazard_does_not_become_evidence_of_safety():
result = summary([row(hazard=None, success=False)])[0]
hazards = result['target_any_deployment_hazard']
assert hazards['unknown_targets'] == 1
assert hazards['interval'] == [0.0, 1.0]
@pytest.mark.parametrize('field,value', [('anchors', False), ('split', 'development'), ('horizon', 5),
('execution_mode', 'open_loop'), ('temporal', True),
('escalate', True), ('probe_safety', 'unrestricted'),
('risk_tolerance', 0.1)])
def test_reliability_never_pools_different_experimental_conditions(field, value):
assert len(summary([row(), row(**{field: value})])) == 2
@pytest.mark.parametrize('confidence', [True, 0, 1, -1, float('nan')])
def test_invalid_confidence_rejected(confidence):
with pytest.raises(ValueError, match='confidence'):
summary([], confidence=confidence)
|