tests/test_structure.py

"""Structural quality checks for the Aczen SLM Evaluation Platform.

Owned by the DevOps Lead. Reviews project layout, finds unresolved imports,
and detects circular dependencies WITHOUT importing backend or UI code — pure
static analysis via `ast`, so it is safe to run on CPU-only CI with none of the
heavy dependencies (llama-cpp-python, gradio) installed.

Run as a report:   python tests/test_structure.py
Run as tests:      pytest tests/test_structure.py

ponytail: static AST scan, no framework, no fixtures. Upgrade to real
import-time checks only if a bug ever slips past static analysis.
"""
from __future__ import annotations

import ast
import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent

# PyPI project name -> top-level import name, where they differ.
_IMPORT_NAME_OVERRIDES = {"llama-cpp-python": "llama_cpp"}

# Dev/test-only tools that legitimately appear in tests/ but must NOT be
# shipped in requirements.txt (they never run on the Space).
_DEV_IMPORTS = {"pytest"}

# Deliverables the DevOps Lead owns and guarantees exist.
REQUIRED_PATHS = [
    "README.md",
    "requirements.txt",
    ".gitignore",
    "prompts/system_prompt.txt",
    "prompts/benchmark_prompts.json",
    "prompts/evaluation_prompts.json",
    "rag/__init__.py",
]

# Directories owned by the DevOps Lead — these must be import-clean.
OWNED_DIRS = ("rag", "tests")


# --------------------------------------------------------------------------- #
# Static analysis helpers
# --------------------------------------------------------------------------- #
_EXCLUDE_DIRS = {
    ".git", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
    ".venv", "venv", "env", "build", "dist", ".eggs", "node_modules",
}


def python_files() -> list[Path]:
    """Every .py in the repo (so the check sees all local packages, not a
    hardcoded few), skipping virtualenvs/caches/build dirs."""
    return sorted(
        p for p in ROOT.rglob("*.py")
        if not _EXCLUDE_DIRS & set(p.relative_to(ROOT).parts)
    )


def _module_name(path: Path) -> str:
    rel = path.relative_to(ROOT).with_suffix("")
    parts = list(rel.parts)
    if parts[-1] == "__init__":  # collapse package/__init__.py -> package
        parts = parts[:-1]
    return ".".join(parts)


def local_module_names() -> set[str]:
    names: set[str] = set()
    for path in python_files():
        mod = _module_name(path)
        if mod:
            names.add(mod)
            names.add(mod.split(".")[0])  # top-level package name
    return names


def imported_modules(path: Path) -> set[str]:
    """Full dotted names of absolute imports (relative imports are local)."""
    tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
    names: set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            names.update(a.name for a in node.names)
        elif isinstance(node, ast.ImportFrom) and not node.level and node.module:
            names.add(node.module)
    return names


def requirement_import_names() -> set[str]:
    req = ROOT / "requirements.txt"
    if not req.exists():
        return set()
    names: set[str] = set()
    for raw in req.read_text(encoding="utf-8").splitlines():
        line = raw.strip()
        if not line or line.startswith(("#", "-")):
            continue
        pkg = line.split(";")[0]
        for sep in ("==", ">=", "<=", "~=", "!=", ">", "<", "["):
            pkg = pkg.split(sep)[0]
        pkg = pkg.strip().lower()
        if pkg:
            names.add(_IMPORT_NAME_OVERRIDES.get(pkg, pkg.replace("-", "_")))
    return names


def unresolved_imports() -> dict[str, set[str]]:
    """file -> imported top-level names that are not stdlib, local, or declared."""
    known = (
        set(sys.stdlib_module_names)
        | local_module_names()
        | requirement_import_names()
        | _DEV_IMPORTS
    )
    out: dict[str, set[str]] = {}
    for path in python_files():
        missing = {m.split(".")[0] for m in imported_modules(path)} - known
        if missing:
            out[str(path.relative_to(ROOT)).replace("\\", "/")] = missing
    return out


def import_cycles() -> list[list[str]]:
    """Cycles among local modules only (third-party edges ignored)."""
    local = local_module_names()
    graph: dict[str, set[str]] = {}
    for path in python_files():
        mod = _module_name(path)
        deps: set[str] = set()
        for name in imported_modules(path):
            target = name if name in local else name.split(".")[0]
            if target in local and target != mod:
                deps.add(target)
        graph.setdefault(mod, set()).update(deps)

    cycles: list[list[str]] = []
    WHITE, GREY, BLACK = 0, 1, 2
    color = {n: WHITE for n in graph}
    stack: list[str] = []

    def visit(n: str) -> None:
        color[n] = GREY
        stack.append(n)
        for t in graph.get(n, ()):  # resolve top-package targets to a real node
            nodes = [t] if t in graph else [k for k in graph if k.split(".")[0] == t]
            for node in nodes:
                if node == n:
                    continue
                if color.get(node, BLACK) == GREY:
                    cycles.append(stack[stack.index(node):] + [node])
                elif color.get(node, BLACK) == WHITE:
                    visit(node)
        stack.pop()
        color[n] = BLACK

    for n in list(graph):
        if color[n] == WHITE:
            visit(n)
    return cycles


# --------------------------------------------------------------------------- #
# Tests (deterministic invariants on files the DevOps Lead owns)
# --------------------------------------------------------------------------- #
def test_required_paths_exist():
    missing = [p for p in REQUIRED_PATHS if not (ROOT / p).exists()]
    assert not missing, f"Missing required deliverables: {missing}"


def test_prompt_json_is_valid():
    bench = json.loads((ROOT / "prompts/benchmark_prompts.json").read_text(encoding="utf-8"))
    assert bench["prompts"], "benchmark_prompts.json has no prompts"
    ids = [p["id"] for p in bench["prompts"]]
    assert len(ids) == len(set(ids)), "duplicate benchmark prompt ids"
    for p in bench["prompts"]:
        assert p.get("category") and p.get("prompt"), f"incomplete prompt: {p}"
    ev = json.loads((ROOT / "prompts/evaluation_prompts.json").read_text(encoding="utf-8"))
    assert ev["criteria"] and ev.get("judge_template"), "evaluation rubric incomplete"


def test_no_import_cycles():
    cycles = import_cycles()
    assert not cycles, f"circular imports detected: {cycles}"


def test_owned_dirs_have_no_unresolved_imports():
    # Backend/UI may reference modules still in progress; only gate what we own.
    offenders = {
        f: names
        for f, names in unresolved_imports().items()
        if f.split("/")[0] in OWNED_DIRS
    }
    assert not offenders, f"unresolved imports in owned dirs: {offenders}"


# --------------------------------------------------------------------------- #
# Standalone report: python tests/test_structure.py
# --------------------------------------------------------------------------- #
def main() -> int:
    print("Aczen SLM - project structure review\n" + "=" * 40)
    ok = True

    print("\n[required deliverables]")
    for p in REQUIRED_PATHS:
        exists = (ROOT / p).exists()
        ok = ok and exists
        print(f"  {'OK ' if exists else 'MISSING'}  {p}")

    print("\n[import cycles]")
    cycles = import_cycles()
    if cycles:
        ok = False
        for c in cycles:
            print("  CYCLE: " + " -> ".join(c))
    else:
        print("  none")

    print("\n[unresolved imports]  (not stdlib / local / in requirements.txt)")
    unresolved = unresolved_imports()
    if unresolved:
        for f, names in sorted(unresolved.items()):
            owned = f.split("/")[0] in OWNED_DIRS
            tag = "FIX" if owned else "note"
            print(f"  {tag}: {f} -> {sorted(names)}")
        print("  suggest: add missing packages to requirements.txt, or create "
              "the local module (backend/UI modules may still be in progress).")
    else:
        print("  none")

    print("\nRESULT:", "PASS" if ok else "FAIL")
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main())