"""Structural quality checks for the Aczen SLM Evaluation Platform. Owned by the DevOps Lead. Reviews project layout, finds unresolved imports, and detects circular dependencies WITHOUT importing backend or UI code — pure static analysis via `ast`, so it is safe to run on CPU-only CI with none of the heavy dependencies (llama-cpp-python, gradio) installed. Run as a report: python tests/test_structure.py Run as tests: pytest tests/test_structure.py ponytail: static AST scan, no framework, no fixtures. Upgrade to real import-time checks only if a bug ever slips past static analysis. """ from __future__ import annotations import ast import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent # PyPI project name -> top-level import name, where they differ. _IMPORT_NAME_OVERRIDES = {"llama-cpp-python": "llama_cpp"} # Dev/test-only tools that legitimately appear in tests/ but must NOT be # shipped in requirements.txt (they never run on the Space). _DEV_IMPORTS = {"pytest"} # Deliverables the DevOps Lead owns and guarantees exist. REQUIRED_PATHS = [ "README.md", "requirements.txt", ".gitignore", "prompts/system_prompt.txt", "prompts/benchmark_prompts.json", "prompts/evaluation_prompts.json", "rag/__init__.py", ] # Directories owned by the DevOps Lead — these must be import-clean. OWNED_DIRS = ("rag", "tests") # --------------------------------------------------------------------------- # # Static analysis helpers # --------------------------------------------------------------------------- # _EXCLUDE_DIRS = { ".git", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".venv", "venv", "env", "build", "dist", ".eggs", "node_modules", } def python_files() -> list[Path]: """Every .py in the repo (so the check sees all local packages, not a hardcoded few), skipping virtualenvs/caches/build dirs.""" return sorted( p for p in ROOT.rglob("*.py") if not _EXCLUDE_DIRS & set(p.relative_to(ROOT).parts) ) def _module_name(path: Path) -> str: rel = path.relative_to(ROOT).with_suffix("") parts = list(rel.parts) if parts[-1] == "__init__": # collapse package/__init__.py -> package parts = parts[:-1] return ".".join(parts) def local_module_names() -> set[str]: names: set[str] = set() for path in python_files(): mod = _module_name(path) if mod: names.add(mod) names.add(mod.split(".")[0]) # top-level package name return names def imported_modules(path: Path) -> set[str]: """Full dotted names of absolute imports (relative imports are local).""" tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) names: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): names.update(a.name for a in node.names) elif isinstance(node, ast.ImportFrom) and not node.level and node.module: names.add(node.module) return names def requirement_import_names() -> set[str]: req = ROOT / "requirements.txt" if not req.exists(): return set() names: set[str] = set() for raw in req.read_text(encoding="utf-8").splitlines(): line = raw.strip() if not line or line.startswith(("#", "-")): continue pkg = line.split(";")[0] for sep in ("==", ">=", "<=", "~=", "!=", ">", "<", "["): pkg = pkg.split(sep)[0] pkg = pkg.strip().lower() if pkg: names.add(_IMPORT_NAME_OVERRIDES.get(pkg, pkg.replace("-", "_"))) return names def unresolved_imports() -> dict[str, set[str]]: """file -> imported top-level names that are not stdlib, local, or declared.""" known = ( set(sys.stdlib_module_names) | local_module_names() | requirement_import_names() | _DEV_IMPORTS ) out: dict[str, set[str]] = {} for path in python_files(): missing = {m.split(".")[0] for m in imported_modules(path)} - known if missing: out[str(path.relative_to(ROOT)).replace("\\", "/")] = missing return out def import_cycles() -> list[list[str]]: """Cycles among local modules only (third-party edges ignored).""" local = local_module_names() graph: dict[str, set[str]] = {} for path in python_files(): mod = _module_name(path) deps: set[str] = set() for name in imported_modules(path): target = name if name in local else name.split(".")[0] if target in local and target != mod: deps.add(target) graph.setdefault(mod, set()).update(deps) cycles: list[list[str]] = [] WHITE, GREY, BLACK = 0, 1, 2 color = {n: WHITE for n in graph} stack: list[str] = [] def visit(n: str) -> None: color[n] = GREY stack.append(n) for t in graph.get(n, ()): # resolve top-package targets to a real node nodes = [t] if t in graph else [k for k in graph if k.split(".")[0] == t] for node in nodes: if node == n: continue if color.get(node, BLACK) == GREY: cycles.append(stack[stack.index(node):] + [node]) elif color.get(node, BLACK) == WHITE: visit(node) stack.pop() color[n] = BLACK for n in list(graph): if color[n] == WHITE: visit(n) return cycles # --------------------------------------------------------------------------- # # Tests (deterministic invariants on files the DevOps Lead owns) # --------------------------------------------------------------------------- # def test_required_paths_exist(): missing = [p for p in REQUIRED_PATHS if not (ROOT / p).exists()] assert not missing, f"Missing required deliverables: {missing}" def test_prompt_json_is_valid(): bench = json.loads((ROOT / "prompts/benchmark_prompts.json").read_text(encoding="utf-8")) assert bench["prompts"], "benchmark_prompts.json has no prompts" ids = [p["id"] for p in bench["prompts"]] assert len(ids) == len(set(ids)), "duplicate benchmark prompt ids" for p in bench["prompts"]: assert p.get("category") and p.get("prompt"), f"incomplete prompt: {p}" ev = json.loads((ROOT / "prompts/evaluation_prompts.json").read_text(encoding="utf-8")) assert ev["criteria"] and ev.get("judge_template"), "evaluation rubric incomplete" def test_no_import_cycles(): cycles = import_cycles() assert not cycles, f"circular imports detected: {cycles}" def test_owned_dirs_have_no_unresolved_imports(): # Backend/UI may reference modules still in progress; only gate what we own. offenders = { f: names for f, names in unresolved_imports().items() if f.split("/")[0] in OWNED_DIRS } assert not offenders, f"unresolved imports in owned dirs: {offenders}" # --------------------------------------------------------------------------- # # Standalone report: python tests/test_structure.py # --------------------------------------------------------------------------- # def main() -> int: print("Aczen SLM - project structure review\n" + "=" * 40) ok = True print("\n[required deliverables]") for p in REQUIRED_PATHS: exists = (ROOT / p).exists() ok = ok and exists print(f" {'OK ' if exists else 'MISSING'} {p}") print("\n[import cycles]") cycles = import_cycles() if cycles: ok = False for c in cycles: print(" CYCLE: " + " -> ".join(c)) else: print(" none") print("\n[unresolved imports] (not stdlib / local / in requirements.txt)") unresolved = unresolved_imports() if unresolved: for f, names in sorted(unresolved.items()): owned = f.split("/")[0] in OWNED_DIRS tag = "FIX" if owned else "note" print(f" {tag}: {f} -> {sorted(names)}") print(" suggest: add missing packages to requirements.txt, or create " "the local module (backend/UI modules may still be in progress).") else: print(" none") print("\nRESULT:", "PASS" if ok else "FAIL") return 0 if ok else 1 if __name__ == "__main__": raise SystemExit(main())