RepoReaper / tests /test_evaluation_correctness_layer.py
GitHub Actions Bot
deploy: auto-inject hf config & sync
30f75ba
Raw History Blame Contribute Delete
19.8 kB
import json
import os
import asyncio
from datetime import datetime
import pytest
from evaluation.analyze_eval_results import EvaluationAnalyzer
from evaluation.data_router import DataRoutingEngine
from evaluation.models import (
AgenticMetrics,
DataQualityTier,
EvaluationResult,
GenerationMetrics,
QueryRewriteMetrics,
RetrievalMetrics,
)
def _make_generation_metrics(
score: float = 0.8,
query: str = "How does this work?",
context: str = "def foo():\n return 1",
answer: str = "A" * 160,
) -> GenerationMetrics:
return GenerationMetrics(
query=query,
retrieved_context=context,
generated_answer=answer,
faithfulness=score,
answer_relevance=score,
answer_completeness=score,
code_correctness=1.0,
)
def _line_count(path: str) -> int:
if not os.path.exists(path):
return 0
with open(path, "r", encoding="utf-8") as f:
return sum(1 for _ in f)
def test_data_quality_tier_single_source_routing(tmp_path):
router = DataRoutingEngine(output_dir=str(tmp_path))
result = EvaluationResult(
session_id="s1",
query="q1",
repo_url="https://github.com/a/b",
timestamp=datetime.now(),
generation_metrics=_make_generation_metrics(score=0.8),
)
# 模拟外部路径手动注入分数与错误 tier
result.overall_score = 0.65
result.data_quality_tier = DataQualityTier.GOLD
tier = router.route_sample(result)
assert tier == DataQualityTier.BRONZE.value
assert result.data_quality_tier == DataQualityTier.BRONZE
assert result.sft_ready is False
assert _line_count(router.positive_samples_file) == 0
assert _line_count(router.negative_samples_file) == 1
def test_data_quality_tier_boundaries_are_canonical():
assert DataQualityTier.from_score(0.9) == DataQualityTier.GOLD
assert DataQualityTier.from_score(0.7) == DataQualityTier.SILVER
assert DataQualityTier.from_score(0.5) == DataQualityTier.BRONZE
assert DataQualityTier.from_score(0.49) == DataQualityTier.REJECTED
assert DataQualityTier.min_score_for(DataQualityTier.SILVER) == 0.7
def test_data_router_statistics_no_dpo_placeholder(tmp_path):
router = DataRoutingEngine(output_dir=str(tmp_path))
stats = router.get_statistics()
assert set(stats.keys()) == {"positive", "negative"}
def test_evaluation_result_to_dict_includes_layer_overall_scores():
query_rewrite = QueryRewriteMetrics(
original_query="auth flow",
rewritten_query="authentication login middleware",
language_detected="en",
keyword_coverage=0.8,
semantic_preservation=0.9,
diversity_score=0.7,
)
retrieval = RetrievalMetrics(
query="auth flow",
top_k=5,
hit_rate=1.0,
recall_at_k=0.8,
precision_at_k=0.6,
mrr=1.0,
context_relevance=0.6,
chunk_integrity=0.8,
retrieval_latency_ms=120.0,
vector_score_avg=0.7,
bm25_score_avg=0.3,
)
generation = _make_generation_metrics(score=0.85)
agentic = AgenticMetrics(
query="auth flow",
tool_selection_accuracy=1.0,
tool_parameter_correctness=1.0,
steps_taken=3,
unnecessary_steps=0,
backtrack_count=0,
success=True,
)
result = EvaluationResult(
session_id="s2",
query="auth flow",
repo_url="https://github.com/a/b",
timestamp=datetime.now(),
query_rewrite_metrics=query_rewrite,
retrieval_metrics=retrieval,
generation_metrics=generation,
agentic_metrics=agentic,
)
result.compute_overall_score()
payload = result.to_dict()
assert payload["query_rewrite"]["overall_score"] == pytest.approx(query_rewrite.overall_score())
assert payload["retrieval"]["overall_score"] == pytest.approx(retrieval.overall_score())
assert payload["generation"]["overall_score"] == pytest.approx(generation.overall_score())
assert payload["agentic"]["overall_score"] == pytest.approx(agentic.overall_score())
assert "dpo_candidate" not in payload
def test_layer_performance_reads_serialized_layer_scores(tmp_path):
result = EvaluationResult(
session_id="s3",
query="q3",
repo_url="https://github.com/a/b",
timestamp=datetime.now(),
generation_metrics=_make_generation_metrics(score=0.9),
)
result.compute_overall_score()
eval_file = tmp_path / "eval_results.jsonl"
with open(eval_file, "w", encoding="utf-8") as f:
f.write(json.dumps(result.to_dict(), ensure_ascii=False) + "\n")
analyzer = EvaluationAnalyzer(eval_results_file=str(eval_file))
layer_stats = analyzer.layer_performance()
assert "generation" in layer_stats
assert layer_stats["generation"]["avg"] == pytest.approx(
result.generation_metrics.overall_score()
)
def test_auto_eval_respects_enabled_flag(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
import app.services.auto_evaluation_service as auto_eval_module
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None)
monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None)
class FakeEvalEngine:
called = 0
async def evaluate_generation(self, **kwargs):
self.called += 1
return _make_generation_metrics()
fake_engine = FakeEvalEngine()
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
service = AutoEvaluationService(
eval_engine=fake_engine,
data_router=router,
config=EvaluationConfig(
enabled=False,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
out = asyncio.run(
service.auto_evaluate(
query="q",
retrieved_context="def x(): pass",
generated_answer="answer long enough",
session_id="sid",
repo_url="https://github.com/a/b",
)
)
assert out is None
assert fake_engine.called == 0
assert not os.path.exists(router.eval_results_file)
def test_needs_review_is_routed_only_after_approve(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
import app.services.auto_evaluation_service as auto_eval_module
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None)
monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None)
class FakeEvalEngine:
async def evaluate_generation(self, **kwargs):
return _make_generation_metrics(
score=0.82,
query=kwargs["query"],
context=kwargs["retrieved_context"],
answer=kwargs["generated_answer"],
)
class TestService(AutoEvaluationService):
async def _ragas_eval(self, query, context, answer):
return 0.1, "mock_ragas"
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
service = TestService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
use_ragas=True,
ragas_sample_rate=1.0,
diff_threshold=0.2,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
tier = asyncio.run(
service.auto_evaluate(
query="how does it work",
retrieved_context="def x(): pass",
generated_answer="A" * 180,
session_id="sid-1",
repo_url="https://github.com/a/b",
)
)
assert tier in {"gold", "silver", "bronze"}
assert len(service.get_review_queue()) == 1
before = _line_count(router.eval_results_file)
service.approve_sample(0)
after = _line_count(router.eval_results_file)
assert len(service.get_review_queue()) == 0
assert before == 0
assert after == 1
def test_needs_review_reject_does_not_route(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
import app.services.auto_evaluation_service as auto_eval_module
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None)
monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None)
class FakeEvalEngine:
async def evaluate_generation(self, **kwargs):
return _make_generation_metrics(
score=0.82,
query=kwargs["query"],
context=kwargs["retrieved_context"],
answer=kwargs["generated_answer"],
)
class TestService(AutoEvaluationService):
async def _ragas_eval(self, query, context, answer):
return 0.1, "mock_ragas"
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
service = TestService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
use_ragas=True,
ragas_sample_rate=1.0,
diff_threshold=0.2,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
asyncio.run(
service.auto_evaluate(
query="how does it work",
retrieved_context="def x(): pass",
generated_answer="A" * 180,
session_id="sid-1",
repo_url="https://github.com/a/b",
)
)
assert len(service.get_review_queue()) == 1
before = _line_count(router.eval_results_file)
service.reject_sample(0)
after = _line_count(router.eval_results_file)
assert before == 0
assert after == 0
assert len(service.get_review_queue()) == 0
def test_rejected_sample_is_persisted_for_audit(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
import app.services.auto_evaluation_service as auto_eval_module
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None)
monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None)
class FakeEvalEngine:
async def evaluate_generation(self, **kwargs):
# 0.30 -> rejected
return _make_generation_metrics(
score=0.3,
query=kwargs["query"],
context=kwargs["retrieved_context"],
answer=kwargs["generated_answer"],
)
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
service = AutoEvaluationService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
use_ragas=False,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
visualize_only=False,
),
)
tier = asyncio.run(
service.auto_evaluate(
query="why rejected",
retrieved_context="def x(): pass",
generated_answer="A" * 180,
session_id="sid-rejected",
repo_url="https://github.com/a/b",
)
)
assert tier == DataQualityTier.REJECTED.value
assert _line_count(router.eval_results_file) == 1
assert _line_count(router.positive_samples_file) == 0
assert _line_count(router.negative_samples_file) == 0
def test_dedupe_key_is_committed_only_after_terminal_success(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
import app.services.auto_evaluation_service as auto_eval_module
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None)
monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None)
class FlakyEvalEngine:
def __init__(self):
self.calls = 0
async def evaluate_generation(self, **kwargs):
self.calls += 1
if self.calls == 1:
raise RuntimeError("transient_failure")
return _make_generation_metrics(
score=0.8,
query=kwargs["query"],
context=kwargs["retrieved_context"],
answer=kwargs["generated_answer"],
)
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
engine = FlakyEvalEngine()
service = AutoEvaluationService(
eval_engine=engine,
data_router=router,
config=EvaluationConfig(
enabled=True,
use_ragas=False,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
visualize_only=False,
),
)
# 第一次失败:不应持久化 dedupe key,应允许重试
first = asyncio.run(
service.auto_evaluate(
query="same-query",
retrieved_context="def x(): pass",
generated_answer="A" * 180,
session_id="sid-dedupe",
repo_url="https://github.com/a/b",
)
)
assert first is None
assert len(service._evaluated_keys) == 0
# 第二次重试应执行成功并落盘 dedupe key
second = asyncio.run(
service.auto_evaluate(
query="same-query",
retrieved_context="def x(): pass",
generated_answer="A" * 180,
session_id="sid-dedupe",
repo_url="https://github.com/a/b",
)
)
assert second in {"gold", "silver", "bronze", "rejected"}
assert engine.calls == 2
assert service._check_duplicate("same-query", "sid-dedupe") is True
def test_retrieval_script_drift_fixed():
with open("evaluation/test_retrieval.py", "r", encoding="utf-8") as f:
content = f.read()
assert "file_list = await get_repo_structure(repo_url)" in content
assert "store.collection.count()" not in content
def test_review_queue_persists_and_approve_by_sample_id_is_idempotent(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
import app.services.auto_evaluation_service as auto_eval_module
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None)
monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None)
class FakeEvalEngine:
async def evaluate_generation(self, **kwargs):
return _make_generation_metrics(
score=0.82,
query=kwargs["query"],
context=kwargs["retrieved_context"],
answer=kwargs["generated_answer"],
)
class TestService(AutoEvaluationService):
async def _ragas_eval(self, query, context, answer):
return 0.1, "mock_ragas"
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
service = TestService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
use_ragas=True,
ragas_sample_rate=1.0,
diff_threshold=0.2,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
asyncio.run(
service.auto_evaluate(
query="how does it work",
retrieved_context="def x(): pass",
generated_answer="A" * 180,
session_id="sid-1",
repo_url="https://github.com/a/b",
)
)
queue = service.get_review_queue()
assert len(queue) == 1
sample_id = queue[0].get("sample_id")
assert sample_id
# 重建 service,验证队列持久化
service_reloaded = TestService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
use_ragas=True,
ragas_sample_rate=1.0,
diff_threshold=0.2,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
reloaded_queue = service_reloaded.get_review_queue()
assert any(item.get("sample_id") == sample_id for item in reloaded_queue)
before = _line_count(router.eval_results_file)
ok1, _ = service_reloaded.approve_sample_by_id(sample_id)
after_first = _line_count(router.eval_results_file)
ok2, _ = service_reloaded.approve_sample_by_id(sample_id)
after_second = _line_count(router.eval_results_file)
assert ok1 is True
assert ok2 is True
assert before == 0
assert after_first == 1
assert after_second == 1
def test_dedupe_cache_persists_across_restart(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.environ.setdefault("LLM_PROVIDER", "openai")
os.environ.setdefault("OPENAI_API_KEY", "dummy")
from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig
class FakeEvalEngine:
async def evaluate_generation(self, **kwargs):
return _make_generation_metrics()
router = DataRoutingEngine(output_dir=str(tmp_path / "sft"))
service = AutoEvaluationService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
assert service._check_duplicate("same-query", "same-session") is False
service_reloaded = AutoEvaluationService(
eval_engine=FakeEvalEngine(),
data_router=router,
config=EvaluationConfig(
enabled=True,
async_evaluation=False,
min_query_length=1,
min_answer_length=1,
require_repo_url=False,
require_code_in_context=False,
),
)
assert service_reloaded._check_duplicate("same-query", "same-session") is True