File size: 4,474 Bytes
547ce6e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | """Citation verifier — the last line of defence against a fabricated reference."""
from __future__ import annotations
import pytest
from app.core.models import CitationStatus
from app.rag.verify import extract_citations, find_uncited_claims, verify_answer
from tests.conftest import make_chunk, make_scored
@pytest.fixture
def evidence():
return [
make_scored(make_chunk(article_no=51, law_label="Labour Law")),
make_scored(make_chunk(article_no=30, law_label="Labour Law")),
make_scored(make_chunk(article_no=25, law_label="Tenancy Law", law_id="dubai-tenancy-law")),
]
class TestExtractCitations:
@pytest.mark.parametrize(
("text", "expected"),
[
("x [Labour Law, Article 51] y", ("Labour Law", 51)),
("x [Labour Law, Art. 51] y", ("Labour Law", 51)),
("x [Labour Law, Art 51] y", ("Labour Law", 51)),
("x [labour law , article (51) ] y", ("labour law", 51)),
("x [Tenancy Law, Article 25] y", ("Tenancy Law", 25)),
],
)
def test_accepts_the_tolerated_forms(self, text: str, expected: tuple[str, int]):
"""A verifier that only recognises perfect formatting silently passes bad output."""
found = extract_citations(text)
assert len(found) == 1
assert (found[0][1], found[0][2]) == expected
def test_ignores_prose_that_is_not_a_citation(self):
assert extract_citations("Article 51 says something without brackets") == []
assert extract_citations("[see the annex]") == []
class TestVerifyAnswer:
def test_citation_present_in_context_is_verified(self, evidence):
report = verify_answer("Gratuity is payable [Labour Law, Article 51].", evidence)
assert report.passed
assert report.verified_count == 1
assert report.unsupported_count == 0
assert report.citations[0].chunk_id is not None
def test_fabricated_article_is_flagged_not_dropped(self, evidence):
"""Article 99 was never retrieved. The answer is flagged, and the citation kept.
Silently deleting it would hand the reader a redacted answer with no way to know
something was removed.
"""
report = verify_answer("It says X [Labour Law, Article 99].", evidence)
assert not report.passed
assert report.unsupported_count == 1
assert report.citations[0].status is CitationStatus.UNSUPPORTED
assert report.citations[0].chunk_id is None
def test_right_article_wrong_law_is_unsupported(self, evidence):
"""Article 51 exists — in the Labour Law, not the Tenancy Law."""
report = verify_answer("See [Tenancy Law, Article 51].", evidence)
assert not report.passed
assert report.unsupported_count == 1
def test_label_matching_ignores_case_and_punctuation(self, evidence):
report = verify_answer("See [labour-law, Article 51].", evidence)
assert report.passed
def test_matches_at_article_level_not_chunk_level(self):
"""A model given chunk 2 of Article 30 and citing Article 30 has cited correctly."""
chunk = make_chunk(article_no=30, text="Labour Law - Article 30 (2/2)\ntail of the article")
report = verify_answer("See [Labour Law, Article 30].", [make_scored(chunk)])
assert report.passed
def test_no_citations_at_all_still_passes_but_reports_uncited(self, evidence):
"""Zero citations is not a fabrication; it is reported as unverified prose."""
report = verify_answer("The law is generally quite protective of workers here.", evidence)
assert report.passed
assert report.verified_count == 0
assert report.uncited_sentences
def test_empty_evidence_makes_every_citation_unsupported(self):
report = verify_answer("See [Labour Law, Article 51].", [])
assert not report.passed
assert report.unsupported_count == 1
class TestUncitedClaims:
def test_short_fragments_are_not_claims(self):
assert find_uncited_claims("Yes. No. Maybe.") == []
def test_cited_sentences_are_not_reported(self):
text = "A worker gets thirty days of annual leave each year [Labour Law, Article 29]."
assert find_uncited_claims(text) == []
def test_uncited_assertion_is_reported(self):
text = "The employer must always pay double wages on public holidays without exception."
assert len(find_uncited_claims(text)) == 1
|