"""Tests for the structuring call: parsing, validation, retry, and failure.""" import json import pytest from buglens import structure from buglens.examples import EXAMPLES from buglens.prompts import STRUCTURE_PROMPT from buglens.schema import BugReport # Field-name JSON (model_dump default) is exactly what the model is asked to emit. VALID_JSON = json.dumps(EXAMPLES["payment"].model_dump()) def test_structure_prompt_enforces_quality_floors() -> None: # Guard the hardening rules so they can't silently regress. assert "Never claim data loss" in STRUCTURE_PROMPT assert 'critical" ONLY' in STRUCTURE_PROMPT assert "missing_info is the most important field" in STRUCTURE_PROMPT assert STRUCTURE_PROMPT.count("at least 3") >= 2 # regression_tests and edge_cases def test_extract_json_strips_fences_and_prose() -> None: fenced = "```json\n{\"a\": 1}\n```" assert structure.extract_json(fenced) == '{"a": 1}' chatty = 'Sure! Here you go:\n{"a": 1}\nHope that helps.' assert structure.extract_json(chatty) == '{"a": 1}' def test_extract_json_raises_when_absent() -> None: with pytest.raises(ValueError): structure.extract_json("no json here") def test_parse_report_round_trips() -> None: report = structure.parse_report(VALID_JSON) assert isinstance(report, BugReport) assert report.app == "Checkout" def test_fallback_report_preserves_quality_floors_without_overclaiming() -> None: observation = """Visible facts: - The screen is a checkout payment page. - The card number field is filled. - The Pay button appears disabled. Not visible: - Browser, OS, environment, API response, and affected users are not visible. """ report = structure.fallback_report( observation, "Pay button stays disabled after entering a valid test card.", ) assert report.severity.value == "high" assert report.app == "Checkout" assert len(report.missing_info) >= 3 assert len(report.regression_tests) >= 3 assert len(report.edge_cases) >= 3 assert len(report.vision_read) >= 3 no_overclaims = f"{report.summary} {report.severity_why}".lower() assert "data loss" not in no_overclaims assert "security" not in no_overclaims assert "privacy" not in no_overclaims assert "model json" not in f"{report.blurb} {report.summary}".lower() def test_fallback_report_stays_medium_when_impact_is_unclear() -> None: report = structure.fallback_report( "Visible facts:\n- A settings screen shows a blank panel.", "", ) assert report.severity.value == "medium" assert report.app == "Application" assert any(not item.known for item in report.env) # ── fakes that return queued decode outputs ── class _FakeTensor: shape = (1, 3) class _FakeInputs(dict): def to(self, _device): return self class _SeqProcessor: def __init__(self, outputs): self._outputs = list(outputs) self.calls = 0 def apply_chat_template(self, messages, **kwargs): self.last_messages = messages return _FakeInputs(input_ids=_FakeTensor()) def decode(self, _ids, **kwargs): out = self._outputs[self.calls] self.calls += 1 return out class _FakeModel: device = "cpu" def generate(self, **kwargs): return [[0, 1, 2, 3]] def test_structure_parses_on_first_try() -> None: processor = _SeqProcessor([VALID_JSON]) report = structure.structure(_FakeModel(), processor, "obs", "note") assert report.app == "Checkout" assert processor.calls == 1 # no retry needed def test_structure_retries_once_then_succeeds() -> None: processor = _SeqProcessor(["not json at all", VALID_JSON]) report = structure.structure(_FakeModel(), processor, "obs") assert isinstance(report, BugReport) assert processor.calls == 2 # retried exactly once def test_structure_raises_after_failed_retry() -> None: processor = _SeqProcessor(["garbage", "{bad json"]) with pytest.raises(structure.StructureError) as exc: structure.structure(_FakeModel(), processor, "obs") assert processor.calls == 2 assert exc.value.raw == "{bad json" # last raw output preserved for the UI