| """Tests for coder execution benchmark dataset coverage.""" |
|
|
| import json |
| from pathlib import Path |
|
|
| from maris_core.text.benchmark import load_chat_benchmark_dataset |
|
|
|
|
| def test_coder_execution_benchmark_dataset_covers_primary_languages() -> None: |
| dataset_path = Path( |
| "/home/runner/work/Maris-MI/Maris-MI/core-python/evals/coder_execution_benchmark.json" |
| ) |
|
|
| cases = load_chat_benchmark_dataset(dataset_path) |
|
|
| languages = {case.execution_language for case in cases if case.execution_language} |
| assert {"python", "typescript", "rust", "sql"}.issubset(languages) |
| assert all(case.level == "ci" for case in cases) |
|
|
|
|
| def test_coder_release_benchmark_dataset_is_execution_and_repo_heavy() -> None: |
| dataset_path = Path( |
| "/home/runner/work/Maris-MI/Maris-MI/core-python/evals/coder_release_benchmark.json" |
| ) |
|
|
| cases = load_chat_benchmark_dataset(dataset_path) |
|
|
| execution_cases = [case for case in cases if case.execution_language] |
| repo_cases = [case for case in cases if "repo-level" in case.tags or "diff" in case.tags] |
| assert len(cases) >= 10 |
| assert len(execution_cases) >= 5 |
| assert repo_cases |
| assert any("multi-file" in case.tags for case in cases) |
| assert any("regression-risk" in case.tags for case in cases) |
| assert any("incident-recovery" in case.tags for case in cases) |
| assert any("debugging" in case.tags for case in cases) |
|
|
|
|
| def test_coder_sample_dataset_covers_repo_diff_and_incident_work() -> None: |
| dataset_path = Path("/home/runner/work/Maris-MI/Maris-MI/data/code/sample.jsonl") |
|
|
| records = [ |
| json.loads(line) |
| for line in dataset_path.read_text(encoding="utf-8").splitlines() |
| if line.strip() |
| ] |
|
|
| task_types = {str(record.get("metadata", {}).get("task", "")).strip() for record in records} |
| repo_context_counts = [len(record.get("repo_context", [])) for record in records] |
| prompts = [str(record.get("prompt", "")) for record in records] |
|
|
| assert {"repo-level", "bugfix", "refactor", "ci-orchestration"}.issubset(task_types) |
| assert max(repo_context_counts) >= 2 |
| assert any("incident" in prompt.lower() for prompt in prompts) |
| assert any(str(record.get("diff", "")).strip() for record in records) |
|
|
