Spaces:
Sleeping
Sleeping
File size: 4,400 Bytes
7b72b2c 9255fb5 7b72b2c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """Tests for Rubric markdown loader: construction validation, hash, permutation."""
from __future__ import annotations
from pathlib import Path
import pytest
from agent_bench.evaluation.judges.base import Rubric
FIXTURES = Path(__file__).parent / "fixtures"
class TestRubricLoading:
def test_load_valid_binary(self):
r = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_binary.md")
assert r.dimension == "groundedness"
assert r.scale == "binary"
assert r.reference_based is True
assert r.abstain_allowed is True
assert len(r.levels) == 2
def test_load_valid_three_point(self):
r = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_three_point.md")
assert r.dimension == "relevance"
assert r.scale == "three_point"
assert len(r.levels) == 3
def test_fenced_code_examples_do_not_break_level_count(self):
"""Regression: the level-pattern regex must skip ``## Score N`` strings
that appear inside fenced code blocks. A binary rubric whose
Example A contains a code-fenced ``## Score 7`` literal should still
load as a 2-level binary rubric, not be rejected with arity mismatch.
"""
r = Rubric.from_markdown_file(
FIXTURES / "rubrics_valid_with_fenced_examples.md"
)
assert r.dimension == "groundedness"
assert r.scale == "binary"
assert len(r.levels) == 2, (
f"fenced ## Score 7 leaked into level count; got {len(r.levels)} levels"
)
class TestRubricValidationErrors:
@pytest.mark.parametrize(
"fixture_name,error_substring",
[
("rubrics_invalid_scale.md", "scale"),
("rubrics_invalid_arity.md", "arity"),
("rubrics_invalid_no_examples.md", "anchored example"),
("rubrics_invalid_no_frontmatter.md", "frontmatter"),
],
)
def test_construction_raises_with_path_and_field(
self, fixture_name: str, error_substring: str
):
path = FIXTURES / fixture_name
with pytest.raises(ValueError) as exc_info:
Rubric.from_markdown_file(path)
msg = str(exc_info.value)
# Error must mention the file path and the field-level reason
assert fixture_name in msg, f"Path missing from error: {msg}"
assert error_substring in msg.lower(), (
f"Expected '{error_substring}' in error message: {msg}"
)
class TestRubricSourceHash:
def test_source_hash_deterministic(self):
r1 = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_binary.md")
r2 = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_binary.md")
assert r1.source_hash == r2.source_hash
# SHA-256 hex is 64 chars
assert len(r1.source_hash) == 64
def test_source_hash_changes_with_content(self):
r1 = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_binary.md")
r2 = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_three_point.md")
assert r1.source_hash != r2.source_hash
class TestRubricPermutation:
def test_render_prompt_seed_0_unchanged(self):
r = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_three_point.md")
prompt = r.render_prompt(level_permutation_seed=0)
# Default: levels in original 0, 1, 2 order
idx0 = prompt.index("Score 0")
idx1 = prompt.index("Score 1")
idx2 = prompt.index("Score 2")
assert idx0 < idx1 < idx2
def test_render_prompt_seed_reproducibility(self):
r = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_three_point.md")
p1 = r.render_prompt(level_permutation_seed=42)
p2 = r.render_prompt(level_permutation_seed=42)
assert p1 == p2
def test_render_prompt_different_seed_different_order(self):
r = Rubric.from_markdown_file(FIXTURES / "rubrics_valid_three_point.md")
# Try several seeds; at least one should produce a non-default order
# (with 3! = 6 permutations, the chance all 5 seeds produce identity
# is (1/6)^5 ≈ 1e-4, negligible)
default = r.render_prompt(level_permutation_seed=0)
differs = any(
r.render_prompt(level_permutation_seed=s) != default
for s in (1, 2, 3, 7, 13)
)
assert differs, "No seed produced a permutation different from default"
|