Spaces:
Runtime error
Runtime error
File size: 8,093 Bytes
5fca0ca 295512e 5fca0ca 295512e 5fca0ca 295512e 5fca0ca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | """Tests for ai_percent contract enforcement: assembly tier, verbatim
ratio target, deterministic stitcher, and prompt-stated slider value.
"""
from __future__ import annotations
from app.generator.postprocess import verbatim_overlap_ratio
from app.generator.prompts import (
build_user_prompt,
top_p_for_ai_involvement,
verbatim_ratio_target,
)
from app.services.generation import _deterministic_assembly_stitch
class TestTopPCurveMonotonic:
def test_assembly_is_tightest(self) -> None:
assert top_p_for_ai_involvement(0) < top_p_for_ai_involvement(50)
assert top_p_for_ai_involvement(0) < top_p_for_ai_involvement(100)
def test_curve_is_monotonic(self) -> None:
values = [top_p_for_ai_involvement(p) for p in (0, 12, 25, 37, 50, 67, 80, 100)]
for earlier, later in zip(values, values[1:]):
assert earlier <= later, f"top_p decreased from {earlier} to {later}"
def test_assembly_below_one(self) -> None:
assert top_p_for_ai_involvement(0) < 1.0
assert top_p_for_ai_involvement(5) < 1.0
class TestVerbatimOverlapRatio:
def test_full_quote_scores_high(self) -> None:
src = "The property is a three-storey semi-detached house in fair condition."
out = "The property is a three-storey semi-detached house in fair condition."
assert verbatim_overlap_ratio(out, [src], n=6) == 1.0
def test_paraphrase_scores_low(self) -> None:
src = "The property is a three-storey semi-detached house in fair condition."
out = "We observed a moderately maintained three-level semi-detached dwelling."
assert verbatim_overlap_ratio(out, [src], n=6) < 0.2
def test_empty_inputs_return_zero(self) -> None:
assert verbatim_overlap_ratio("", ["any"], n=6) == 0.0
assert verbatim_overlap_ratio("any text here please", [], n=6) == 0.0
assert verbatim_overlap_ratio("short", ["short"], n=6) == 0.0
def test_partial_quote_proportional(self) -> None:
src = "The roof covering is in fair condition with no significant defects observed."
out = "The roof covering is in fair condition. We also noticed loose tiles near the ridge."
ratio = verbatim_overlap_ratio(out, [src], n=6)
assert 0.0 < ratio < 1.0
class TestDeterministicStitcher:
BOILERPLATE = (
"The property comprises a traditionally constructed dwelling of brick and "
"block cavity wall construction beneath a pitched and tiled roof. "
"Inspection was carried out from ground level using a non-invasive visual approach."
)
def test_includes_source_wording_verbatim(self) -> None:
out = _deterministic_assembly_stitch(
skeleton="[A]: description.",
bullets=["3 Acacia Avenue, postcode SW1A 1AA", "freehold, vacant"],
paragraph_snippets=[self.BOILERPLATE],
document_snippets=None,
hierarchy_section_snippets=None,
survey_level=3,
redact_references=False,
)
assert "traditionally constructed dwelling" in out
assert "non-invasive visual approach" in out
def test_appends_bullet_notes(self) -> None:
out = _deterministic_assembly_stitch(
skeleton="[A]: description.",
bullets=["3 Acacia Avenue, postcode XX1 1XX", "freehold, vacant"],
paragraph_snippets=[self.BOILERPLATE],
document_snippets=None,
hierarchy_section_snippets=None,
survey_level=3,
redact_references=False,
)
assert "Site-specific observations" in out
assert "Acacia Avenue" in out
def test_redaction_strips_other_property_facts(self) -> None:
ref_with_facts = (
"The property at 25 Cunliffe Road, London SW4 8QE is a Victorian terrace. "
"Inspection was carried out from ground level."
)
out = _deterministic_assembly_stitch(
skeleton="[A]: description.",
bullets=["different property notes"],
paragraph_snippets=[ref_with_facts],
document_snippets=None,
hierarchy_section_snippets=None,
survey_level=3,
redact_references=True,
)
assert "SW4 8QE" not in out
assert "Cunliffe Road" not in out
assert "Inspection was carried out" in out
def test_returns_empty_when_no_boilerplate(self) -> None:
out = _deterministic_assembly_stitch(
skeleton="[A]: description.",
bullets=["only bullets here"],
paragraph_snippets=None,
document_snippets=None,
hierarchy_section_snippets=None,
survey_level=3,
)
assert out == ""
def test_falls_back_to_document_snippets(self) -> None:
out = _deterministic_assembly_stitch(
skeleton="[A]: description.",
bullets=["one bullet"],
paragraph_snippets=None,
document_snippets=[self.BOILERPLATE],
hierarchy_section_snippets=None,
survey_level=3,
redact_references=False,
)
assert "traditionally constructed dwelling" in out
def test_no_synthetic_advice_phrases(self) -> None:
out = _deterministic_assembly_stitch(
skeleton="[A]",
bullets=["bullet text alpha"],
paragraph_snippets=[self.BOILERPLATE],
document_snippets=None,
hierarchy_section_snippets=None,
survey_level=3,
redact_references=False,
)
synthetic_phrases = [
"we recommend",
"should be replaced",
"in our professional opinion",
"it is advisable",
]
for phrase in synthetic_phrases:
assert phrase not in out.lower(), f"synthetic phrase {phrase!r} leaked into stitcher output"
class TestVerbatimRatioTarget:
def test_zero_percent_demands_full_verbatim(self) -> None:
target, floor = verbatim_ratio_target(0)
assert target == 1.0
assert floor >= 0.9
def test_full_ai_no_verbatim_floor(self) -> None:
target, floor = verbatim_ratio_target(100)
assert target == 0.0
assert floor == 0.0
def test_target_decreases_monotonically(self) -> None:
targets = [verbatim_ratio_target(p)[0] for p in (0, 25, 50, 75, 100)]
for earlier, later in zip(targets, targets[1:]):
assert earlier >= later
def test_floor_never_above_target(self) -> None:
for p in (10, 25, 50, 75):
target, floor = verbatim_ratio_target(p)
assert 0.0 <= floor <= target
class TestPromptStatesSliderRatio:
"""The user-visible prompt MUST state the slider's target ratio so the
LLM has a concrete number to hit (and so we can audit drift)."""
def test_assembly_prompt_quotes_slider_value(self) -> None:
prompt = build_user_prompt(
skeleton="A",
bullets=["alpha"],
paragraph_snippets=["The property comprises a traditionally constructed dwelling."],
ai_percent=0,
survey_level=3,
reference_only_context=True,
)
assert "0%" in prompt
assert "verbatim" in prompt.lower()
def test_low_tier_prompt_quotes_slider_value(self) -> None:
prompt = build_user_prompt(
skeleton="A",
bullets=["alpha"],
paragraph_snippets=["Inspection was carried out from ground level."],
ai_percent=25,
survey_level=3,
reference_only_context=True,
)
assert "25%" in prompt
assert "75%" in prompt
def test_mid_tier_prompt_quotes_slider_value(self) -> None:
prompt = build_user_prompt(
skeleton="A",
bullets=["alpha"],
paragraph_snippets=["Inspection was carried out from ground level."],
ai_percent=50,
survey_level=3,
reference_only_context=True,
)
assert "50%" in prompt
|