Spaces:
Running
Running
Download tests/test_evaluation_correctness_layer.py from realdexter/RepoReaper: direct link, hf CLI and curl.
- Browser
- Download file 19.8 kB
-
https://huggingface.co/spaces/realdexter/RepoReaper/resolve/main/tests/test_evaluation_correctness_layer.py
- Command line
-
hf download hf://spaces/realdexter/RepoReaper/tests/test_evaluation_correctness_layer.py
-
curl -L -o test_evaluation_correctness_layer.py https://huggingface.co/spaces/realdexter/RepoReaper/resolve/main/tests/test_evaluation_correctness_layer.py
19.8 kB
| import json | |
| import os | |
| import asyncio | |
| from datetime import datetime | |
| import pytest | |
| from evaluation.analyze_eval_results import EvaluationAnalyzer | |
| from evaluation.data_router import DataRoutingEngine | |
| from evaluation.models import ( | |
| AgenticMetrics, | |
| DataQualityTier, | |
| EvaluationResult, | |
| GenerationMetrics, | |
| QueryRewriteMetrics, | |
| RetrievalMetrics, | |
| ) | |
| def _make_generation_metrics( | |
| score: float = 0.8, | |
| query: str = "How does this work?", | |
| context: str = "def foo():\n return 1", | |
| answer: str = "A" * 160, | |
| ) -> GenerationMetrics: | |
| return GenerationMetrics( | |
| query=query, | |
| retrieved_context=context, | |
| generated_answer=answer, | |
| faithfulness=score, | |
| answer_relevance=score, | |
| answer_completeness=score, | |
| code_correctness=1.0, | |
| ) | |
| def _line_count(path: str) -> int: | |
| if not os.path.exists(path): | |
| return 0 | |
| with open(path, "r", encoding="utf-8") as f: | |
| return sum(1 for _ in f) | |
| def test_data_quality_tier_single_source_routing(tmp_path): | |
| router = DataRoutingEngine(output_dir=str(tmp_path)) | |
| result = EvaluationResult( | |
| session_id="s1", | |
| query="q1", | |
| repo_url="https://github.com/a/b", | |
| timestamp=datetime.now(), | |
| generation_metrics=_make_generation_metrics(score=0.8), | |
| ) | |
| # 模拟外部路径手动注入分数与错误 tier | |
| result.overall_score = 0.65 | |
| result.data_quality_tier = DataQualityTier.GOLD | |
| tier = router.route_sample(result) | |
| assert tier == DataQualityTier.BRONZE.value | |
| assert result.data_quality_tier == DataQualityTier.BRONZE | |
| assert result.sft_ready is False | |
| assert _line_count(router.positive_samples_file) == 0 | |
| assert _line_count(router.negative_samples_file) == 1 | |
| def test_data_quality_tier_boundaries_are_canonical(): | |
| assert DataQualityTier.from_score(0.9) == DataQualityTier.GOLD | |
| assert DataQualityTier.from_score(0.7) == DataQualityTier.SILVER | |
| assert DataQualityTier.from_score(0.5) == DataQualityTier.BRONZE | |
| assert DataQualityTier.from_score(0.49) == DataQualityTier.REJECTED | |
| assert DataQualityTier.min_score_for(DataQualityTier.SILVER) == 0.7 | |
| def test_data_router_statistics_no_dpo_placeholder(tmp_path): | |
| router = DataRoutingEngine(output_dir=str(tmp_path)) | |
| stats = router.get_statistics() | |
| assert set(stats.keys()) == {"positive", "negative"} | |
| def test_evaluation_result_to_dict_includes_layer_overall_scores(): | |
| query_rewrite = QueryRewriteMetrics( | |
| original_query="auth flow", | |
| rewritten_query="authentication login middleware", | |
| language_detected="en", | |
| keyword_coverage=0.8, | |
| semantic_preservation=0.9, | |
| diversity_score=0.7, | |
| ) | |
| retrieval = RetrievalMetrics( | |
| query="auth flow", | |
| top_k=5, | |
| hit_rate=1.0, | |
| recall_at_k=0.8, | |
| precision_at_k=0.6, | |
| mrr=1.0, | |
| context_relevance=0.6, | |
| chunk_integrity=0.8, | |
| retrieval_latency_ms=120.0, | |
| vector_score_avg=0.7, | |
| bm25_score_avg=0.3, | |
| ) | |
| generation = _make_generation_metrics(score=0.85) | |
| agentic = AgenticMetrics( | |
| query="auth flow", | |
| tool_selection_accuracy=1.0, | |
| tool_parameter_correctness=1.0, | |
| steps_taken=3, | |
| unnecessary_steps=0, | |
| backtrack_count=0, | |
| success=True, | |
| ) | |
| result = EvaluationResult( | |
| session_id="s2", | |
| query="auth flow", | |
| repo_url="https://github.com/a/b", | |
| timestamp=datetime.now(), | |
| query_rewrite_metrics=query_rewrite, | |
| retrieval_metrics=retrieval, | |
| generation_metrics=generation, | |
| agentic_metrics=agentic, | |
| ) | |
| result.compute_overall_score() | |
| payload = result.to_dict() | |
| assert payload["query_rewrite"]["overall_score"] == pytest.approx(query_rewrite.overall_score()) | |
| assert payload["retrieval"]["overall_score"] == pytest.approx(retrieval.overall_score()) | |
| assert payload["generation"]["overall_score"] == pytest.approx(generation.overall_score()) | |
| assert payload["agentic"]["overall_score"] == pytest.approx(agentic.overall_score()) | |
| assert "dpo_candidate" not in payload | |
| def test_layer_performance_reads_serialized_layer_scores(tmp_path): | |
| result = EvaluationResult( | |
| session_id="s3", | |
| query="q3", | |
| repo_url="https://github.com/a/b", | |
| timestamp=datetime.now(), | |
| generation_metrics=_make_generation_metrics(score=0.9), | |
| ) | |
| result.compute_overall_score() | |
| eval_file = tmp_path / "eval_results.jsonl" | |
| with open(eval_file, "w", encoding="utf-8") as f: | |
| f.write(json.dumps(result.to_dict(), ensure_ascii=False) + "\n") | |
| analyzer = EvaluationAnalyzer(eval_results_file=str(eval_file)) | |
| layer_stats = analyzer.layer_performance() | |
| assert "generation" in layer_stats | |
| assert layer_stats["generation"]["avg"] == pytest.approx( | |
| result.generation_metrics.overall_score() | |
| ) | |
| def test_auto_eval_respects_enabled_flag(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| import app.services.auto_evaluation_service as auto_eval_module | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None) | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None) | |
| class FakeEvalEngine: | |
| called = 0 | |
| async def evaluate_generation(self, **kwargs): | |
| self.called += 1 | |
| return _make_generation_metrics() | |
| fake_engine = FakeEvalEngine() | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| service = AutoEvaluationService( | |
| eval_engine=fake_engine, | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=False, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| out = asyncio.run( | |
| service.auto_evaluate( | |
| query="q", | |
| retrieved_context="def x(): pass", | |
| generated_answer="answer long enough", | |
| session_id="sid", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| assert out is None | |
| assert fake_engine.called == 0 | |
| assert not os.path.exists(router.eval_results_file) | |
| def test_needs_review_is_routed_only_after_approve(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| import app.services.auto_evaluation_service as auto_eval_module | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None) | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None) | |
| class FakeEvalEngine: | |
| async def evaluate_generation(self, **kwargs): | |
| return _make_generation_metrics( | |
| score=0.82, | |
| query=kwargs["query"], | |
| context=kwargs["retrieved_context"], | |
| answer=kwargs["generated_answer"], | |
| ) | |
| class TestService(AutoEvaluationService): | |
| async def _ragas_eval(self, query, context, answer): | |
| return 0.1, "mock_ragas" | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| service = TestService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| use_ragas=True, | |
| ragas_sample_rate=1.0, | |
| diff_threshold=0.2, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| tier = asyncio.run( | |
| service.auto_evaluate( | |
| query="how does it work", | |
| retrieved_context="def x(): pass", | |
| generated_answer="A" * 180, | |
| session_id="sid-1", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| assert tier in {"gold", "silver", "bronze"} | |
| assert len(service.get_review_queue()) == 1 | |
| before = _line_count(router.eval_results_file) | |
| service.approve_sample(0) | |
| after = _line_count(router.eval_results_file) | |
| assert len(service.get_review_queue()) == 0 | |
| assert before == 0 | |
| assert after == 1 | |
| def test_needs_review_reject_does_not_route(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| import app.services.auto_evaluation_service as auto_eval_module | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None) | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None) | |
| class FakeEvalEngine: | |
| async def evaluate_generation(self, **kwargs): | |
| return _make_generation_metrics( | |
| score=0.82, | |
| query=kwargs["query"], | |
| context=kwargs["retrieved_context"], | |
| answer=kwargs["generated_answer"], | |
| ) | |
| class TestService(AutoEvaluationService): | |
| async def _ragas_eval(self, query, context, answer): | |
| return 0.1, "mock_ragas" | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| service = TestService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| use_ragas=True, | |
| ragas_sample_rate=1.0, | |
| diff_threshold=0.2, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| asyncio.run( | |
| service.auto_evaluate( | |
| query="how does it work", | |
| retrieved_context="def x(): pass", | |
| generated_answer="A" * 180, | |
| session_id="sid-1", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| assert len(service.get_review_queue()) == 1 | |
| before = _line_count(router.eval_results_file) | |
| service.reject_sample(0) | |
| after = _line_count(router.eval_results_file) | |
| assert before == 0 | |
| assert after == 0 | |
| assert len(service.get_review_queue()) == 0 | |
| def test_rejected_sample_is_persisted_for_audit(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| import app.services.auto_evaluation_service as auto_eval_module | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None) | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None) | |
| class FakeEvalEngine: | |
| async def evaluate_generation(self, **kwargs): | |
| # 0.30 -> rejected | |
| return _make_generation_metrics( | |
| score=0.3, | |
| query=kwargs["query"], | |
| context=kwargs["retrieved_context"], | |
| answer=kwargs["generated_answer"], | |
| ) | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| service = AutoEvaluationService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| use_ragas=False, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| visualize_only=False, | |
| ), | |
| ) | |
| tier = asyncio.run( | |
| service.auto_evaluate( | |
| query="why rejected", | |
| retrieved_context="def x(): pass", | |
| generated_answer="A" * 180, | |
| session_id="sid-rejected", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| assert tier == DataQualityTier.REJECTED.value | |
| assert _line_count(router.eval_results_file) == 1 | |
| assert _line_count(router.positive_samples_file) == 0 | |
| assert _line_count(router.negative_samples_file) == 0 | |
| def test_dedupe_key_is_committed_only_after_terminal_success(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| import app.services.auto_evaluation_service as auto_eval_module | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None) | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None) | |
| class FlakyEvalEngine: | |
| def __init__(self): | |
| self.calls = 0 | |
| async def evaluate_generation(self, **kwargs): | |
| self.calls += 1 | |
| if self.calls == 1: | |
| raise RuntimeError("transient_failure") | |
| return _make_generation_metrics( | |
| score=0.8, | |
| query=kwargs["query"], | |
| context=kwargs["retrieved_context"], | |
| answer=kwargs["generated_answer"], | |
| ) | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| engine = FlakyEvalEngine() | |
| service = AutoEvaluationService( | |
| eval_engine=engine, | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| use_ragas=False, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| visualize_only=False, | |
| ), | |
| ) | |
| # 第一次失败:不应持久化 dedupe key,应允许重试 | |
| first = asyncio.run( | |
| service.auto_evaluate( | |
| query="same-query", | |
| retrieved_context="def x(): pass", | |
| generated_answer="A" * 180, | |
| session_id="sid-dedupe", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| assert first is None | |
| assert len(service._evaluated_keys) == 0 | |
| # 第二次重试应执行成功并落盘 dedupe key | |
| second = asyncio.run( | |
| service.auto_evaluate( | |
| query="same-query", | |
| retrieved_context="def x(): pass", | |
| generated_answer="A" * 180, | |
| session_id="sid-dedupe", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| assert second in {"gold", "silver", "bronze", "rejected"} | |
| assert engine.calls == 2 | |
| assert service._check_duplicate("same-query", "sid-dedupe") is True | |
| def test_retrieval_script_drift_fixed(): | |
| with open("evaluation/test_retrieval.py", "r", encoding="utf-8") as f: | |
| content = f.read() | |
| assert "file_list = await get_repo_structure(repo_url)" in content | |
| assert "store.collection.count()" not in content | |
| def test_review_queue_persists_and_approve_by_sample_id_is_idempotent(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| import app.services.auto_evaluation_service as auto_eval_module | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "add_event", lambda *args, **kwargs: None) | |
| monkeypatch.setattr(auto_eval_module.tracing_service, "record_score", lambda *args, **kwargs: None) | |
| class FakeEvalEngine: | |
| async def evaluate_generation(self, **kwargs): | |
| return _make_generation_metrics( | |
| score=0.82, | |
| query=kwargs["query"], | |
| context=kwargs["retrieved_context"], | |
| answer=kwargs["generated_answer"], | |
| ) | |
| class TestService(AutoEvaluationService): | |
| async def _ragas_eval(self, query, context, answer): | |
| return 0.1, "mock_ragas" | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| service = TestService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| use_ragas=True, | |
| ragas_sample_rate=1.0, | |
| diff_threshold=0.2, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| asyncio.run( | |
| service.auto_evaluate( | |
| query="how does it work", | |
| retrieved_context="def x(): pass", | |
| generated_answer="A" * 180, | |
| session_id="sid-1", | |
| repo_url="https://github.com/a/b", | |
| ) | |
| ) | |
| queue = service.get_review_queue() | |
| assert len(queue) == 1 | |
| sample_id = queue[0].get("sample_id") | |
| assert sample_id | |
| # 重建 service,验证队列持久化 | |
| service_reloaded = TestService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| use_ragas=True, | |
| ragas_sample_rate=1.0, | |
| diff_threshold=0.2, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| reloaded_queue = service_reloaded.get_review_queue() | |
| assert any(item.get("sample_id") == sample_id for item in reloaded_queue) | |
| before = _line_count(router.eval_results_file) | |
| ok1, _ = service_reloaded.approve_sample_by_id(sample_id) | |
| after_first = _line_count(router.eval_results_file) | |
| ok2, _ = service_reloaded.approve_sample_by_id(sample_id) | |
| after_second = _line_count(router.eval_results_file) | |
| assert ok1 is True | |
| assert ok2 is True | |
| assert before == 0 | |
| assert after_first == 1 | |
| assert after_second == 1 | |
| def test_dedupe_cache_persists_across_restart(tmp_path, monkeypatch): | |
| monkeypatch.chdir(tmp_path) | |
| os.environ.setdefault("LLM_PROVIDER", "openai") | |
| os.environ.setdefault("OPENAI_API_KEY", "dummy") | |
| from app.services.auto_evaluation_service import AutoEvaluationService, EvaluationConfig | |
| class FakeEvalEngine: | |
| async def evaluate_generation(self, **kwargs): | |
| return _make_generation_metrics() | |
| router = DataRoutingEngine(output_dir=str(tmp_path / "sft")) | |
| service = AutoEvaluationService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| assert service._check_duplicate("same-query", "same-session") is False | |
| service_reloaded = AutoEvaluationService( | |
| eval_engine=FakeEvalEngine(), | |
| data_router=router, | |
| config=EvaluationConfig( | |
| enabled=True, | |
| async_evaluation=False, | |
| min_query_length=1, | |
| min_answer_length=1, | |
| require_repo_url=False, | |
| require_code_in_context=False, | |
| ), | |
| ) | |
| assert service_reloaded._check_duplicate("same-query", "same-session") is True | |