test: expand test suite to 150+ tests and 80%+ coverage

- Added 30 parametrized scenario validation tests (97 total cases)
- Implemented shared fixtures in conftest.py with in-memory DB override
- Added persistence layer tests for SQLite and leaderboard logic
- Expanded grader tests with edge cases for line tolerance and penalties
- Added API error path and pagination tests
- Fixed logic bug in SecurityGrader keyword thresholding
- Configured pytest-cov with 80% coverage enforcement

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.coverage.run]
+source = ["codereview_env", "app"]
+omit = ["tests/*", "scripts/*", "*/migrations/*"]
+
+[tool.coverage.report]
+fail_under = 80
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "if __name__ == .__main__.:",
+    "raise NotImplementedError",
+]
+
+[tool.pylint.messages_control]
+disable = ["C0114", "C0115", "C0116"]   # disable missing-docstring warnings

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests
+addopts = --tb=short -q
+markers =
+    slow: marks tests as slow (use -m "not slow" to skip)
+    integration: marks integration tests

requirements.txt

@@ -11,3 +11,4 @@ slowapi==0.1.9
 python-dotenv==1.0.1
 sqlmodel==0.0.16
 aiosqlite==0.20.0
+pytest-cov==4.1.0

tests/conftest.py

@@ -0,0 +1,44 @@
+import pytest
+from fastapi.testclient import TestClient
+from sqlmodel import SQLModel, Session, create_engine
+from sqlmodel.pool import StaticPool
+
+from app import app
+from codereview_env.database import get_session
+from codereview_env.env import CodeReviewEnv
+from codereview_env.models import TaskId, Action, ActionType, Severity, Category, Verdict
+
+@pytest.fixture(name="session")
+def session_fixture():
+    """In-memory SQLite session for tests."""
+    engine = create_engine(
+        "sqlite://",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    SQLModel.metadata.create_all(engine)
+    with Session(engine) as session:
+        yield session
+
+@pytest.fixture(name="client")
+def client_fixture(session):
+    """TestClient with DB dependency override."""
+    def get_session_override():
+        yield session
+    app.dependency_overrides[get_session] = get_session_override
+    client = TestClient(app)
+    yield client
+    app.dependency_overrides.clear()
+
+@pytest.fixture
+def env():
+    return CodeReviewEnv()
+
+@pytest.fixture
+def approve_action():
+    return Action(action_type=ActionType.APPROVE, body="LGTM", verdict=Verdict.LGTM)
+
+@pytest.fixture
+def request_changes_action():
+    return Action(action_type=ActionType.REQUEST_CHANGES, body="Issues found",
+                  verdict=Verdict.REQUEST_CHANGES)

tests/test_api.py

@@ -62,3 +62,68 @@ def test_api_invalid_episode():
         "body": "hello"
     })
     assert response.status_code == 404
+
+def test_api_health_fields(client):
+    resp = client.get("/health")
+    data = resp.json()
+    assert "active_episodes" in data
+    assert "auth_enabled" in data
+    assert "env" in data
+
+def test_api_reset_invalid_task(client):
+    resp = client.post("/reset", json={"task_id": "invalid_task", "seed": 0})
+    assert resp.status_code == 422
+
+def test_api_step_invalid_action_type(client):
+    reset_resp = client.post("/reset", json={"task_id": "bug_detection", "seed": 0})
+    episode_id = reset_resp.json()["episode_id"]
+    resp = client.post(f"/step/{episode_id}", json={"action_type": "not_valid", "body": "x"})
+    assert resp.status_code == 422
+
+def test_api_result_after_completion(client):
+    """Result endpoint should return persisted data for completed episodes."""
+    reset_resp = client.post("/reset", json={"task_id": "bug_detection", "seed": 0})
+    episode_id = reset_resp.json()["episode_id"]
+    # Complete the episode
+    client.post(f"/step/{episode_id}", json={
+        "action_type": "approve", "body": "LGTM", "verdict": "lgtm"
+    })
+    # Result must be available
+    result_resp = client.get(f"/result/{episode_id}")
+    assert result_resp.status_code == 200
+    assert result_resp.json()["final_score"] >= 0
+
+def test_api_stats_endpoint(client):
+    resp = client.get("/stats")
+    assert resp.status_code == 200
+    assert "total_episodes" in resp.json()
+
+@pytest.mark.parametrize("task_id", ["bug_detection", "security_audit", "architectural_review"])
+def test_api_full_workflow_all_tasks(client, task_id):
+    reset = client.post("/reset", json={"task_id": task_id, "seed": 1})
+    assert reset.status_code == 200
+    episode_id = reset.json()["episode_id"]
+    
+    step = client.post(f"/step/{episode_id}", json={
+        "action_type": "approve", "body": "LGTM", "verdict": "lgtm"
+    })
+    assert step.status_code == 200
+    assert step.json()["done"] is True
+
+def test_api_leaderboard_pagination(client):
+    # Submit 3 entries
+    for i, score in enumerate([0.9, 0.7, 0.5]):
+        client.post("/submit", json={
+            "agent_name": f"agent_{i}", "task_id": "bug_detection",
+            "score": score, "seed": i
+        })
+    
+    # Test limit
+    resp = client.get("/leaderboard?task_id=bug_detection&limit=2")
+    assert resp.status_code == 200
+    data = resp.json()
+    assert len(data["entries"]) == 2
+    assert data["total"] >= 3
+    
+    # Test ordering (best first)
+    assert data["entries"][0]["score"] >= data["entries"][1]["score"]

tests/test_database.py

@@ -0,0 +1,65 @@
+from sqlmodel import Session
+from codereview_env.database import save_episode, get_episode, submit_leaderboard, get_leaderboard_db, get_stats
+from codereview_env.models import EpisodeResult, TaskId, ActionRecord, ActionType
+
+def make_result(episode_id="test-ep-1", score=0.85):
+    return EpisodeResult(
+        episode_id=episode_id,
+        task_id=TaskId.BUG_DETECTION,
+        scenario_hash="bug_001",
+        seed=0,
+        final_score=score,
+        steps_taken=3,
+        issues_found=1,
+        issues_total=1,
+        noise_penalties=0,
+        terminated_reason="terminal_action",
+        history=[ActionRecord(action_type=ActionType.APPROVE, body="LGTM")]
+    )
+
+def test_save_and_get_episode(session):
+    result = make_result()
+    save_episode(session, result)
+    record = get_episode(session, "test-ep-1")
+    assert record is not None
+    assert record.final_score == 0.85
+    assert record.scenario_hash == "bug_001"
+
+def test_get_nonexistent_episode(session):
+    record = get_episode(session, "does-not-exist")
+    assert record is None
+
+def test_episode_history_serialized(session):
+    result = make_result()
+    save_episode(session, result)
+    record = get_episode(session, result.episode_id)
+    import json
+    history = json.loads(record.history_json)
+    assert len(history) == 1
+    assert history[0]["action_type"] == "approve"
+
+def test_leaderboard_submit_and_rank(session):
+    rank = submit_leaderboard(session, "agent_a", "bug_detection", 0.9, 0)
+    assert rank == 1
+    rank2 = submit_leaderboard(session, "agent_b", "bug_detection", 0.7, 1)
+    assert rank2 == 2
+
+def test_leaderboard_ordering(session):
+    submit_leaderboard(session, "low", "security_audit", 0.3, 0)
+    submit_leaderboard(session, "high", "security_audit", 0.95, 1)
+    submit_leaderboard(session, "mid", "security_audit", 0.6, 2)
+    entries, total = get_leaderboard_db(session, "security_audit")
+    assert total == 3
+    assert entries[0].agent_name == "high"
+    assert entries[0].score == 0.95
+
+def test_get_stats_empty(session):
+    stats = get_stats(session)
+    assert stats["total_episodes"] == 0
+
+def test_get_stats_populated(session):
+    save_episode(session, make_result("ep1", 0.9))
+    save_episode(session, make_result("ep2", 0.5))
+    stats = get_stats(session)
+    assert stats["total_episodes"] == 2
+    assert abs(stats["avg_score"] - 0.7) < 0.001

tests/test_env.py

@@ -193,3 +193,59 @@ def test_arch_task_runs_to_completion():
     ))
     final = env.get_final_result()
     assert final.final_score > 0
+
+@pytest.mark.parametrize("task_id", list(TaskId))
+def test_env_reset_all_tasks(task_id, env):
+    """Reset must work for all three task types."""
+    result = env.reset(task_id, seed=0)
+    assert result.task_id == task_id
+    assert result.observation.noise_budget == 5
+
+@pytest.mark.parametrize("task_id,expected_max_steps", [
+    (TaskId.BUG_DETECTION, 10),
+    (TaskId.SECURITY_AUDIT, 15),
+    (TaskId.ARCHITECTURAL_REVIEW, 20),
+])
+def test_env_max_steps_per_task(task_id, expected_max_steps, env):
+    result = env.reset(task_id, seed=0)
+    assert result.observation.max_steps == expected_max_steps
+
+def test_env_step_raises_when_done(env, approve_action):
+    """Calling step on a done episode must raise ValueError."""
+    env.reset(TaskId.BUG_DETECTION, seed=0)
+    env.step(approve_action)
+    with pytest.raises(ValueError):
+        env.step(approve_action)
+
+def test_env_history_recorded(env):
+    """All steps should appear in final result history."""
+    env.reset(TaskId.BUG_DETECTION, seed=0)
+    from codereview_env.models import Action, ActionType
+    for _ in range(3):
+        env.step(Action(action_type=ActionType.ASK_QUESTION, body="question"))
+    env.step(Action(action_type=ActionType.APPROVE, body="LGTM", verdict=Verdict.LGTM))
+    result = env.get_final_result()
+    assert result.steps_taken == 4
+    assert len(result.history) == 4
+
+def test_env_get_final_result_score_clamped(env, approve_action):
+    """Final score must always be in [0, 1]."""
+    env.reset(TaskId.BUG_DETECTION, seed=0)
+    env.step(approve_action)
+    result = env.get_final_result()
+    # Check that score is a float and within [0, 1]
+    assert isinstance(result.final_score, float)
+    assert 0.0 <= result.final_score <= 1.0
+
+@pytest.mark.parametrize("task_id", list(TaskId))
+@pytest.mark.parametrize("seed", [0, 3, 7])
+def test_env_full_episode_completes(task_id, seed, env):
+    """Full episodes must always reach a terminal state."""
+    env.reset(task_id, seed=seed)
+    from codereview_env.models import Action, ActionType, Verdict
+    # Just skip to terminal
+    action = Action(action_type=ActionType.APPROVE, body="LGTM", verdict=Verdict.LGTM)
+    result = env.step(action)
+    assert result.done is True
+    final = env.get_final_result()
+    assert final.terminated_reason == "terminal_action"

tests/test_graders.py

@@ -72,3 +72,146 @@ def test_arch_grader_verdict():
     # issue_score = 1.0, verdict_score = 0.0, quality_score = 0.0
     # score = 0.6 * 1.0 + 0.2 * 0.0 + 0.0 = 0.6
     assert score == 0.6
+
+# ─── Bug Grader Edge Cases ─────────────────────────────
+
+def test_bug_grader_partial_match():
+    """Matching some but not all issues."""
+    scenario = Scenario(
+        task_id=TaskId.BUG_DETECTION, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.BUG, severity=Severity.HIGH,
+                             filename="f1", line_number=10, description="d1", keywords=["k1"]),
+            GroundTruthIssue(id="2", category=Category.BUG, severity=Severity.LOW,
+                             filename="f2", line_number=20, description="d2", keywords=["k2"]),
+        ],
+        hash="test"
+    )
+    history = [
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="k1",
+                     filename="f1", line_number=10, category=Category.BUG, severity=Severity.HIGH)
+    ]
+    score = grade_bug_detection(scenario, history)
+    assert 0.0 < score < 1.0, f"Partial match should give intermediate score, got {score}"
+
+def test_bug_grader_line_tolerance():
+    """Issue flagged within ±3 lines should match."""
+    scenario = Scenario(
+        task_id=TaskId.BUG_DETECTION, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.BUG, severity=Severity.MEDIUM,
+                             filename="f1", line_number=10, description="d", keywords=["bug"])
+        ],
+        hash="test"
+    )
+    # Flag at line 12 (within ±3)
+    history = [
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="bug found here",
+                     filename="f1", line_number=12, category=Category.BUG, severity=Severity.MEDIUM)
+    ]
+    score = grade_bug_detection(scenario, history)
+    assert score > 0.0, "Line within tolerance should match"
+
+def test_bug_grader_line_out_of_tolerance():
+    """Issue flagged outside ±3 lines should NOT match."""
+    scenario = Scenario(
+        task_id=TaskId.BUG_DETECTION, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.BUG, severity=Severity.MEDIUM,
+                             filename="f1", line_number=10, description="d", keywords=["bug"])
+        ],
+        hash="test"
+    )
+    # Flag at line 15 (outside ±3)
+    history = [
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="bug found here",
+                     filename="f1", line_number=15, category=Category.BUG, severity=Severity.MEDIUM)
+    ]
+    score = grade_bug_detection(scenario, history)
+    assert score == 0.0, "Line outside tolerance should not match"
+
+def test_bug_grader_false_positives_penalized():
+    """Multiple FP flags should reduce score."""
+    scenario = Scenario(
+        task_id=TaskId.BUG_DETECTION, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.BUG, severity=Severity.MEDIUM,
+                             filename="f1", line_number=10, description="d", keywords=["real"])
+        ],
+        hash="test"
+    )
+    history = [
+        # One correct flag
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="real bug",
+                     filename="f1", line_number=10, category=Category.BUG, severity=Severity.MEDIUM),
+        # Three false positives
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="fp1",
+                     filename="nowhere", line_number=999, category=Category.BUG, severity=Severity.LOW),
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="fp2",
+                     filename="nowhere", line_number=998, category=Category.BUG, severity=Severity.LOW),
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="fp3",
+                     filename="nowhere", line_number=997, category=Category.BUG, severity=Severity.LOW),
+    ]
+    perfect_score = 1.0
+    score = grade_bug_detection(scenario, history)
+    assert score < perfect_score, "FP flags should reduce score below perfect"
+
+# ─── Security Grader Edge Cases ─────────────────────────
+
+def test_security_grader_perfect():
+    scenario = Scenario(
+        task_id=TaskId.SECURITY_AUDIT, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.SECURITY, severity=Severity.CRITICAL,
+                             filename="f1", line_number=10, description="d", keywords=["sql", "injection"])
+        ],
+        hash="test"
+    )
+    history = [
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body="sql injection vulnerability",
+                     filename="f1", line_number=10, category=Category.SECURITY, severity=Severity.CRITICAL)
+    ]
+    score = grade_security_audit(scenario, history)
+    assert score == 1.0
+
+def test_security_grader_empty_history():
+    scenario = Scenario(
+        task_id=TaskId.SECURITY_AUDIT, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.SECURITY, severity=Severity.HIGH,
+                             filename="f1", line_number=5, description="d", keywords=["k1"])
+        ],
+        hash="test"
+    )
+    assert grade_security_audit(scenario, []) == 0.0
+
+# ─── Arch Grader Edge Cases ─────────────────────────────
+
+def test_arch_grader_correct_verdict():
+    scenario = Scenario(
+        task_id=TaskId.ARCHITECTURAL_REVIEW, pr_title="t", pr_description="t",
+        files_changed=[],
+        ground_truth_issues=[
+            GroundTruthIssue(id="1", category=Category.ARCHITECTURE, severity=Severity.HIGH,
+                             filename="f1", line_number=10, description="d",
+                             keywords=["god class", "single responsibility"],
+                             required_verdict=Verdict.REQUEST_CHANGES)
+        ],
+        hash="test"
+    )
+    # Correct verdict
+    body = "This is a god class violating single responsibility principle and needs major refactoring"
+    history = [
+        ActionRecord(action_type=ActionType.FLAG_ISSUE, body=body,
+                     filename="f1", line_number=10, category=Category.ARCHITECTURE, severity=Severity.HIGH),
+        ActionRecord(action_type=ActionType.REQUEST_CHANGES, body="Needs refactoring",
+                     verdict=Verdict.REQUEST_CHANGES)
+    ]
+    score = grade_architectural_review(scenario, history)
+    assert score > 0.6, f"Correct verdict should score well, got {score}"

tests/test_scenarios.py

@@ -0,0 +1,59 @@
+import pytest
+from codereview_env.scenarios import get_scenario, all_scenarios
+from codereview_env.models import TaskId, Severity
+
+# ── All 30 scenarios loadable ──────────────────────────────────────────────
+
+def test_all_scenarios_count():
+    assert len(all_scenarios()) == 30
+
+@pytest.mark.parametrize("task_id,expected_count", [
+    (TaskId.BUG_DETECTION, 10),
+    (TaskId.SECURITY_AUDIT, 10),
+    (TaskId.ARCHITECTURAL_REVIEW, 10),
+])
+def test_scenario_count_per_task(task_id, expected_count):
+    scenarios = [s for s in all_scenarios() if s.task_id == task_id]
+    assert len(scenarios) == expected_count
+
+@pytest.mark.parametrize("task_id", list(TaskId))
+@pytest.mark.parametrize("seed", range(10))
+def test_scenario_loadable(task_id, seed):
+    """Every seed for every task must return a valid scenario."""
+    s = get_scenario(task_id, seed)
+    assert s is not None
+    assert s.hash != ""
+    assert len(s.ground_truth_issues) >= 1
+    assert len(s.files_changed) >= 1
+
+@pytest.mark.parametrize("task_id", list(TaskId))
+@pytest.mark.parametrize("seed", range(10))
+def test_scenario_has_valid_ground_truth(task_id, seed):
+    s = get_scenario(task_id, seed)
+    for issue in s.ground_truth_issues:
+        assert len(issue.keywords) >= 2, f"Issue {issue.id} needs >= 2 keywords"
+        assert issue.filename != ""
+        assert issue.line_number > 0
+        assert issue.severity in list(Severity)
+
+@pytest.mark.parametrize("task_id", list(TaskId))
+@pytest.mark.parametrize("seed", range(10))
+def test_scenario_has_valid_diff(task_id, seed):
+    s = get_scenario(task_id, seed)
+    for f in s.files_changed:
+        assert "+++" in f.patch or "---" in f.patch or "@@ " in f.patch, \
+            f"Scenario {s.hash}: file {f.filename} patch is not a valid diff"
+
+def test_scenario_hashes_unique():
+    hashes = [s.hash for s in all_scenarios()]
+    assert len(hashes) == len(set(hashes)), "Duplicate scenario hashes found"
+
+def test_scenario_seed_determinism():
+    s1 = get_scenario(TaskId.BUG_DETECTION, 0)
+    s2 = get_scenario(TaskId.BUG_DETECTION, 0)
+    assert s1.hash == s2.hash, "Same seed must return same scenario"
+
+def test_scenario_seed_wraps():
+    s_seed0 = get_scenario(TaskId.BUG_DETECTION, 0)
+    s_seed10 = get_scenario(TaskId.BUG_DETECTION, 10)
+    assert s_seed0.hash == s_seed10.hash, "Seed 10 should wrap to same as seed 0"