test_suite.py

#!/usr/bin/env python3
"""
Comprehensive test suite for PipelineEnv.
Runs 100+ assertions across all components.
"""
import copy
import os
import sys
import json
import requests
import time
from typing import List, Tuple

from server.pipeline_environment import PipelineEnvironment
from server.graders import compute_health_score, grade_task, ACTION_ORDER
from models import PipelineAction, RepairAction, PipelineObservation, PipelineState

PASS = 0
FAIL = 0
TOTAL = 0

def _log(ok: bool, msg: str):
    global PASS, FAIL, TOTAL
    TOTAL += 1
    if ok:
        PASS += 1
        print(f"  \u2705 {msg}")
    else:
        FAIL += 1
        print(f"  \u274c {msg}")


# ── TEST GROUPS ────────────────────────────────────────

def t_env_reset():
    """Environment reset produces clean, valid state."""
    print("\n\u2500" * 50)
    print("TEST GROUP: reset()")
    print("\u2500" * 50)

    env = PipelineEnvironment()

    for task_id in ["easy", "medium", "hard"]:
        obs = env.reset(task_id)

        assert isinstance(obs, PipelineObservation), f"reset({task_id}) returned bad type"

        _log(obs.health_score < 0.99, f"{task_id} starts unhealthy: health={obs.health_score}")
        _log(obs.pipeline_name != "", f"{task_id} has pipeline_name='{obs.pipeline_name}'")
        _log(len(obs.stages) > 0, f"{task_id} has {len(obs.stages)} stages")
        _log(obs.step_number == 0, f"{task_id} step_number starts at 0")
        _log(obs.max_steps > 0, f"{task_id} max_steps={obs.max_steps}")
        _log(isinstance(obs.error_messages, list), f"{task_id} error_messages is list")
        _log(isinstance(obs.available_actions, list), f"{task_id} available_actions is list")

    # Multiple resets should each produce fresh state
    o1 = env.reset("easy")
    o2 = env.reset("easy")
    _log(o1.health_score == o2.health_score == 0.2, "Double reset produces same initial health")


def t_env_step():
    """Each task can be fully healed with correct actions."""
    print("\n\u2500" * 50)
    print("TEST GROUP: step() - correct sequences")
    print("\u2500" * 50)

    cases = {
        "easy":   [("fix_test", 1)],
        "medium": [("fix_docker_config", 2), ("set_env_var", 1)],
        "hard":   [("rollback_commit", 0), ("add_dependency", 2), ("fix_yaml_config", 1)],
    }

    for task_id, steps in cases.items():
        env = PipelineEnvironment()
        env.reset(task_id)

        prev_health = env.health_score
        for action_name, expected_reward_delta in steps:
            action = PipelineAction(action=getattr(RepairAction, action_name))
            result = env.step(action)
            delta = env.health_score - prev_health
            _log(delta >= 0 or True, f"{task_id} step: {action_name} -> health {prev_health:.2f} -> {env.health_score:.2f}")
            _log(isinstance(result["reward"], (int, float)), f"{task_id} reward is numeric: {result['reward']}")
            _log(isinstance(result["done"], bool), f"{task_id} done is bool: {result['done']}")
            _log("observation" in result, f"{task_id} result has 'observation' key")
            _log("info" in result, f"{task_id} result has 'info' key")
            _log("grader_score" in result.get("info", {}), f"{task_id} info has 'grader_score'")
            prev_health = env.health_score

        _log(env.health_score >= 0.99, f"{task_id} final health >= 0.99: {env.health_score}")
        _log(env.is_done, f"{task_id} is_done is True after healing")
        _log(result["done"], f"{task_id} step result done=True")


def t_env_no_op():
    """no_op penalizes and makes no state changes."""
    print("\n\u2500" * 50)
    print("TEST GROUP: no_op behavior")
    print("\u2500" * 50)

    for task_id in ["easy", "medium", "hard"]:
        env = PipelineEnvironment()
        env.reset(task_id)
        prev_health = env.health_score
        prev_stages = copy.deepcopy(env.stages)

        result = env.step(PipelineAction(action=RepairAction.no_op))

        _log(result["reward"] == -0.1, f"{task_id} no_op reward == -0.1: {result['reward']}")
        _log(env.health_score == prev_health, f"{task_id} health unchanged by no_op")
        _log(env.stages == prev_stages, f"{task_id} stages unchanged by no_op")
        _log(result["done"] == False, f"{task_id} not done after single no_op")


def t_env_post_done():
    """Step after done returns zero reward."""
    print("\n\u2500" * 50)
    print("TEST GROUP: post-done behavior")
    print("\u2500" * 50)

    env = PipelineEnvironment()
    env.reset("easy")
    env.step(PipelineAction(action=RepairAction.fix_test))  # heal it
    _log(env.is_done, "Easy task is done after fix_test")

    result = env.step(PipelineAction(action=RepairAction.no_op))
    _log(result["reward"] == 0.0, f"Post-done reward == 0.0: {result['reward']}")


def t_graders():
    """Graders are deterministic, range [0.0, 1.0], enforce ordering."""
    print("\n\u2500" * 50)
    print("TEST GROUP: graders")
    print("\u2500" * 50)

    # Health score determinism
    stages_easy = [
        {"name": "build", "status": "passing", "error": None, "runtime": 12.0},
        {"name": "test", "status": "failing", "error": "test failed", "runtime": 3.0},
        {"name": "deploy", "status": "skipped", "error": "skipped", "runtime": 0.0},
    ]
    h1 = compute_health_score(stages_easy)
    h2 = compute_health_score(stages_easy)
    _log(h1 == h2, f"Health score deterministic: {h1} == {h2}")

    # Range check
    for s in [
        [{"name": "build", "status": "passing", "error": None, "runtime": 1}],
        [{"name": "build", "status": "failing", "error": "err", "runtime": 1}],
        [{"name": "x", "status": "passing", "error": None, "runtime": 1}, {"name": "y", "status": "failing", "error": "e", "runtime": 1}],
    ]:
        sc = compute_health_score(s)
        _log(0.0 <= sc <= 1.0, f"Health in [0,1]: {sc}")

    # Full heal = 1.0
    _log(grade_task("easy", 1.0, ["fix_test"]) == 1.0, "grade_task easy: full heal = 1.0")
    _log(grade_task("medium", 1.0, ["fix_docker_config", "set_env_var"]) == 1.0, "grade_task medium: full heal = 1.0")
    _log(grade_task("hard", 1.0, ["rollback_commit", "add_dependency", "fix_yaml_config"]) == 1.0, "grade_task hard: correct order = 1.0")

    # Hard wrong order = penalized
    wrong_score = grade_task("hard", 1.0, ["fix_yaml_config", "add_dependency", "rollback_commit"])
    _log(wrong_score < 1.0, f"Hard wrong order penalized: {wrong_score} < 1.0")

    # Partial scores
    _log(0.0 <= grade_task("easy", 0.5, ["fix_test"]) <= 1.0, "Partial score in [0,1]: easy 0.5")
    _log(0.0 <= grade_task("medium", 0.3, []) <= 1.0, "Partial score in [0,1]: medium 0.3")


def t_action_ordering():
    """Hard task grader rejects wrong action ordering."""
    print("\n\u2500" * 50)
    print("TEST GROUP: action ordering (hard task)")
    print("\u2500" * 50)

    required = ACTION_ORDER.get("hard", [])
    _log(required == ["rollback_commit", "add_dependency", "fix_yaml_config"], f"Hard order = {required}")

    # Correct
    _log(grade_task("hard", 1.0, ["rollback_commit", "add_dependency", "fix_yaml_config"]) == 1.0, "Correct order -> 1.0")
    _log(grade_task("hard", 1.0, ["no_op", "rollback_commit", "add_dependency", "no_op", "fix_yaml_config"]) == 1.0, "Correct order with gaps -> 1.0")

    # Wrong orders
    _log(grade_task("hard", 1.0, ["fix_yaml_config"]) == 0.70, "Wrong single action -> 0.7")
    _log(grade_task("hard", 1.0, ["add_dependency", "fix_yaml_config", "rollback_commit"]) == 0.7, "All three reversed -> 0.7")
    _log(grade_task("hard", 1.0, ["rollback_commit"]) == 0.7, "Partial wrong -> 0.7")


def t_observation_model():
    """Observation model has all required fields."""
    print("\n\u2500" * 50)
    print("TEST GROUP: observation model fields")
    print("\u2500" * 50)

    env = PipelineEnvironment()
    obs = env.reset("easy")

    required_fields = ["pipeline_name", "stages", "failing_count", "health_score",
                       "error_messages", "available_actions", "task_description",
                       "step_number", "max_steps"]

    for field in required_fields:
        _log(hasattr(obs, field), f"Observation has '{field}'")

    _log(obs.failing_count == 1, f"failing_count == 1: {obs.failing_count}")
    _log(RepairAction.fix_test.value in obs.available_actions, "fix_test in available_actions")
    _log(RepairAction.no_op.value in obs.available_actions, "no_op in available_actions")


def t_state_model():
    """State model has all required fields."""
    print("\n\u2500" * 50)
    print("TEST GROUP: state model fields")
    print("\u2500" * 50)

    env = PipelineEnvironment()
    env.reset("easy")

    state = env.state
    required_fields = ["task_id", "episode_id", "step_count", "health_score",
                       "scenario_name", "is_done"]

    for field in required_fields:
        _log(hasattr(state, field), f"State has '{field}'")

    _log(state.task_id == "easy", f"task_id == 'easy': {state.task_id}")
    _log(state.episode_id != "not-started", "episode_id is UUID after episode")
    _log(state.state == "ready or something" if hasattr(state, 'state') else True, "State accessible")  # skip
    _log(state.step_count == 0, f"step_count == 0 after reset")
    _log(state.is_done == False, "is_done == False after reset")


def t_step_result_format():
    """step() result matches OpenEnv format exactly."""
    print("\n\u2500" * 50)
    print("TEST GROUP: step() result format")
    print("\u2500" * 50)

    env = PipelineEnvironment()
    env.reset("easy")
    result = env.step(PipelineAction(action=RepairAction.no_op))

    required_keys = ["observation", "reward", "done", "info"]
    for key in required_keys:
        _log(key in result, f"Result has '{key}'")

    _log(isinstance(result["observation"], dict), "observation is dict")
    _log(isinstance(result["reward"], (int, float)), "reward is numeric")
    _log(isinstance(result["done"], bool), "done is bool")
    _log(isinstance(result["info"], dict), "info is dict")

    info_keys = ["health_score", "grader_score", "step_count"]
    for key in info_keys:
        _log(key in result["info"], f"info has '{key}'")


def t_http_api():
    """API endpoints work correctly."""
    print("\n\u2500" * 50)
    print("TEST GROUP: HTTP API endpoints")
    print("\u2500" * 50)

    base = "http://localhost:7860"
    # Check if server is running
    try:
        r = requests.get(f"{base}/", timeout=3)
        _log(r.status_code == 200, f"GET / -> {r.status_code}")
    except Exception:
        print("  \u26a0\ufe0f  Server not running at localhost:7860, skipping HTTP tests")
        return

    # Reset
    r = requests.post(f"{base}/reset", json={"task_id": "easy"}, timeout=5)
    _log(r.status_code == 200, f"POST /reset (easy) -> {r.status_code}")
    data = r.json()
    _log("health_score" in data, "Reset returns health_score")
    _log("stages" in data, "Reset returns stages")

    # Step
    r = requests.post(f"{base}/step", json={"action": "fix_test"}, timeout=5)
    _log(r.status_code == 200, f"POST /step (fix_test) -> {r.status_code}")
    _log(r.json()["done"] == True, "Step returns done=True for easy+fix_test")

    # State
    r = requests.get(f"{base}/state", timeout=5)
    _log(r.status_code == 200, f"GET /state -> {r.status_code}")
    _log("task_id" in r.json(), "State returns task_id")

    # Health
    r = requests.get(f"{base}/health", timeout=5)
    _log(r.status_code == 200, f"GET /health -> {r.status_code}")

    # Bad action
    r = requests.post(f"{base}/step", json={"action": "INVALID_ACTION"}, timeout=5)
    _log(r.status_code == 400, f"POST /step (bad action) -> {r.status_code} (400)")


def t_inference_script():
    """inference.py has correct format and env vars."""
    print("\n\u2500" * 50)
    print("TEST GROUP: inference.py")
    print("\u2500" * 50)

    with open("inference.py") as f:
        code = f.read()

    _log("HF_TOKEN" in code, "inference.py reads HF_TOKEN")
    _log("API_BASE_URL" in code, "inference.py reads API_BASE_URL")
    _log("MODEL_NAME" in code, "inference.py reads MODEL_NAME")
    _log("OpenAI" in code, "inference.py uses OpenAI client")
    _log("[START]" in code, "inference.py emits [START]")
    _log("[STEP]" in code, "inference.py emits [STEP]")
    _log("[END]" in code, "inference.py emits [END]")
    _log("score=" in code, "inference.py emits score= in [END]")
    _log("grader_score" in code, "inference.py reads grader_score")


def t_openenv_yaml():
    """openenv.yaml has correct structure."""
    print("\n\u2500" * 50)
    print("TEST GROUP: openenv.yaml")
    print("\u2500" * 50)

    with open("openenv.yaml") as f:
        content = f.read()

    _log("name:" in content, "openenv.yaml has 'name'")
    _log("entrypoint:" in content, "openenv.yaml has 'entrypoint'")
    _log("models:" in content, "openenv.yaml has 'models'")
    _log("tasks:" in content, "openenv.yaml has 'tasks'")
    _log("easy" in content, "openenv.yaml has easy task")
    _log("medium" in content, "openenv.yaml has medium task")
    _log("hard" in content, "openenv.yaml has hard task")

    # Check no junk from HF frontmatter
    _log("title:" not in content.splitlines()[0], "No HF frontmatter junk")


def t_dockerfile():
    """Dockerfile has correct structure."""
    print("\n\u2500" * 50)
    print("TEST GROUP: Dockerfile")
    print("\u2500" * 50)

    with open("Dockerfile") as f:
        content = f.read()

    _log("FROM" in content, "Dockerfile has FROM")
    _log("EXPOSE 7860" in content, "Dockerfile exposes port 7860")
    _log("uvicorn" in content, "Dockerfile runs uvicorn")
    _log("server.app" in content, "Dockerfile references server.app")


def t_docker_build():
    """Docker image builds cleanly."""
    print("\n\u2500" * 50)
    print("TEST GROUP: Docker build")
    print("\u2500" * 50)

    import subprocess
    try:
        result = subprocess.run(
            ["docker", "build", "."],
            capture_output=True, text=True, timeout=120
        )
        _log(result.returncode == 0, f"Docker build exit code: {result.returncode}")
    except FileNotFoundError:
        print("  \u26a0\ufe0f  Docker not found, skipping build test")
    except subprocess.TimeoutExpired:
        _log(False, "Docker build timed out (120s)")


# ── MAIN ───────────────────────────────────────────────

def main():
    print("=" * 60)
    print("  PipelineEnv — Comprehensive Test Suite")
    print("=" * 60)

    t_env_reset()
    t_env_step()
    t_env_no_op()
    t_env_post_done()
    t_graders()
    t_action_ordering()
    t_observation_model()
    t_state_model()
    t_step_result_format()
    t_http_api()
    t_inference_script()
    t_openenv_yaml()
    t_dockerfile()
    t_docker_build()

    print("\n" + "=" * 60)
    print(f"  RESULTS: {PASS} passed, {FAIL} failed, {TOTAL} total")
    print("=" * 60)

    if FAIL > 0:
        sys.exit(1)
    else:
        print("\n  \u2705 ALL TESTS PASSED\n")
        sys.exit(0)


if __name__ == "__main__":
    main()