File size: 2,476 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | from __future__ import annotations
import unittest
from agent_harness.robustness_experiment import (
RobustnessExperimentError,
distractor_severity,
robustness_shift,
synthetic_distractor_sources,
)
from agent_harness.specs import TaskSpec
from agent_harness.syntax_index import parse_go_file
def task() -> TaskSpec:
return TaskSpec(
schema_version=1,
task_id="TASK_CR_999",
repository_url="https://gitlab.com/gitlab-org/gitlab-runner.git",
base_commit="0" * 40,
gold_commit="1" * 40,
language="go",
statement="Cache the role ARN for the docker autoscaler executor.",
gold_patch="gold.patch",
test_patch="test.patch",
gold_files=("executors/docker/autoscaler.go",),
gold_symbols=("executors/docker/autoscaler.go::RoleARN",),
fail_to_pass_tests=("TestRoleARN",),
pass_to_pass_tests=("TestExisting",),
difficulty="test",
provenance="unit test",
validation_status="end_to_end_ready",
)
class RobustnessExperimentTests(unittest.TestCase):
def test_seed_to_dose_mapping_is_frozen(self) -> None:
self.assertEqual([distractor_severity(seed) for seed in (0, 1, 2)], [1, 5, 10])
with self.assertRaises(RobustnessExperimentError):
distractor_severity(3)
def test_distractors_are_nested_deterministic_and_parseable(self) -> None:
one = synthetic_distractor_sources(task(), 1)
ten = synthetic_distractor_sources(task(), 10)
self.assertEqual(one, ten[:1])
self.assertEqual(len({source.path for source in ten}), 10)
self.assertTrue(all(parse_go_file(source.path, source.text) for source in ten))
self.assertTrue(all("role ARN" in source.text for source in ten))
def test_shift_uses_censoring_for_missing_gold(self) -> None:
baseline = {"file_recall_at_10": 1.0, "mrr": 0.5, "ndcg_at_10": 0.7, "first_gold_rank": 2}
perturbed = {"file_recall_at_10": 0.0, "mrr": 0.0, "ndcg_at_10": 0.0, "first_gold_rank": None}
shift = robustness_shift(
baseline,
perturbed,
["other.go", "gold.go"],
["distractor.go"],
["gold.go"],
missing_rank=201,
)
self.assertEqual(shift["first_gold_rank_displacement_censored"], 199)
self.assertTrue(shift["lost_all_top_10_gold"])
self.assertEqual(shift["baseline_gold_top_10_retention"], 0.0)
|