epfo-circulars / tests /test_answer_eval.py
gauravmeena0708's picture
Upload folder using huggingface_hub
3a29659 verified
Raw History Blame Contribute Delete
4.74 kB
"""The answer evaluation scores by fixed rules, and its cases point at real, in-date sources."""
from datetime import date
import json
from pathlib import Path
import re
import sys
ROOT = Path(__file__).parents[1]
sys.path.insert(0, str(ROOT / 'eval/answers'))
import checks
from policy_history import PolicyHistory
from policy_registry import registered_policies
CASES = json.loads((ROOT / 'eval/answers/cases.json').read_text(encoding='utf-8'))['cases']
SEARCH = {
'policy_context': {'policy_id': 'dearness'},
'sources': [{'document_id': 'dr-2025-04-17'}, {'document_id': 'dr-2025-10-13'}, {'document_id': None}],
}
CASE = {
'expect_policy': 'dearness', 'must_cite': ['dr-2025-10-13'],
'must_state': [{'fact': '58 per cent', 'any': [r'\b58 ?%', r'\b58 per ?cent']}],
'must_not_state': [{'claim': '60 per cent', 'any': [r'\b60 ?%']}],
}
def test_good_answer_passes():
answer = 'Dearness relief is **58 %** of basic pension from 1 July 2025 [2].'
assert checks.check_retrieval(CASE, SEARCH) == []
assert checks.check_answer(CASE, SEARCH, answer) == []
def test_each_rule_reports_its_own_failure():
def failures(answer, case=CASE, search=SEARCH):
return checks.check_answer(case, search, answer)
assert failures('') == ['empty answer']
assert 'no numbered citation' in failures('Dearness relief is 58%.')
assert 'does not cite dr-2025-10-13' in failures('Dearness relief is 58% [1].')
assert 'cites sources that do not exist: [7]' in failures('Dearness relief is 58% [2][7].')
assert 'missing: 58 per cent' in failures('Dearness relief was revised [2].')
assert 'states: 60 per cent' in failures('It is 58% [2], later 60%.')
assert any('interface does not link' in f for f in failures('Dearness relief is 58% 【2†source】.'))
# The unlinked style still counts towards the source check, so only one failure is reported for it.
assert 'does not cite dr-2025-10-13' not in failures('Dearness relief is 58% 【2】.')
def test_same_circular_from_the_general_index_counts_as_the_citation():
search = {'sources': [
{'document_id': 'dr-2025-10-13', 'metadata': {'circular_no': 'No.HRD/1(71)/2020/Misc./Pt./1445'}},
{'document_id': None, 'metadata': {'circular_no': 'No. HRD/1(71)/2020/Misc./Pt./1445'}},
{'document_id': None, 'metadata': {'circular_no': 'Another/1'}},
]}
assert checks.check_answer(CASE, search, 'Dearness relief is 58% [2].') == []
assert 'does not cite dr-2025-10-13' in checks.check_answer(CASE, search, 'Dearness relief is 58% [3].')
def test_not_found_answers_need_no_citation():
case = {'citation_optional': True, 'must_state': [{'fact': 'not found', 'any': ['not (?:been )?declared']}]}
assert checks.check_answer(case, SEARCH, 'A rate has not been declared for that year.') == []
def test_route_and_missing_source_are_reported_without_an_answer():
wrong = {'policy_context': None, 'sources': [{'document_id': None}]}
assert checks.check_retrieval(CASE, wrong) == [
'routed to standard search, expected dearness', 'source not retrieved: dr-2025-10-13']
assert checks.check_retrieval({'expect_policy': None}, wrong) == []
def test_answer_language_is_checked():
hindi = 'पेंशन दावा किसी भी अनुसूचित वाणिज्यिक बैंक की किसी भी शाखा के लिए संसाधित किया जा सकता है [1]।'
case = {'language': 'Hindi'}
assert checks.check_answer(case, SEARCH, hindi) == []
assert any('not mainly in Hindi' in f for f in checks.check_answer(case, SEARCH, 'Any branch of any bank [1].'))
assert any('not mainly in English' in f for f in checks.check_answer({}, SEARCH, hindi))
def test_cases_name_real_sources_issued_by_the_evidence_date():
policies = {policy.id: policy for policy in registered_policies()}
histories = {}
assert len({case['id'] for case in CASES}) == len(CASES)
for case in CASES:
assert case['review'] in ('draft', 'reviewed')
assert case['source']
for group in case.get('must_state', []) + case.get('must_not_state', []):
assert group['any']
for pattern in group['any']:
re.compile(pattern)
policy = case.get('expect_policy')
if policy is None:
assert not case.get('must_cite')
continue
history = histories.setdefault(policy, PolicyHistory(policies[policy].path))
for document in case.get('must_cite', []):
issued = history.documents[document]['issued_on']
assert issued <= case.get('as_of', date.today().isoformat()), case['id']