api/tests/run_all_tests.py

#!/usr/bin/env python3
"""
Run all DocGenie API test suites sequentially and collect results.

Usage (from the FYP project root):
    uv run python docgenie/api/tests/run_all_tests.py

Or with the venv activated:
    python docgenie/api/tests/run_all_tests.py
"""
import subprocess
import sys
import json
import pathlib
import datetime

HERE    = pathlib.Path(__file__).parent
ARTIFACTS = HERE / "artifacts"
ARTIFACTS.mkdir(exist_ok=True)

SUITES = [
    {
        "name": "functional",
        "path": str(HERE / "functional"),
        "label": "Functional Testing (Unit Testing)",
    },
    {
        "name": "performance",
        "path": str(HERE / "performance"),
        "label": "Non-Functional Testing (Performance Testing)",
    },
    {
        "name": "reliability",
        "path": str(HERE / "reliability"),
        "label": "Non-Functional Testing (Reliability Testing)",
    },
]


def run_suite(suite: dict) -> dict:
    log_file = ARTIFACTS / f"{suite['name']}_pytest.log"
    json_file = ARTIFACTS / f"{suite['name']}_results.json"

    cmd = [
        sys.executable, "-m", "pytest",
        suite["path"],
        "-v",
        "--tb=short",
        f"--json-report",
        f"--json-report-file={json_file}",
        "-s",
    ]

    print(f"\n{'='*72}")
    print(f"  Running: {suite['label']}")
    print(f"{'='*72}")

    with open(log_file, "w") as lf:
        result = subprocess.run(cmd, capture_output=False, stdout=lf,
                                stderr=subprocess.STDOUT, text=True)

    # Read JSON report if generated
    counts = {"passed": 0, "failed": 0, "error": 0, "skipped": 0, "total": 0}
    test_rows = []
    summary_line = ""
    if json_file.exists():
        try:
            data = json.loads(json_file.read_text())
            summary = data.get("summary", {})
            counts = {
                "passed":  summary.get("passed",  0),
                "failed":  summary.get("failed",  0),
                "error":   summary.get("errors",  0),
                "skipped": summary.get("skipped", 0),
                "total":   summary.get("total",   0),
            }
            for test in data.get("tests", []):
                test_rows.append({
                    "nodeid": test["nodeid"],
                    "outcome": test["outcome"].upper(),
                    "duration_s": round(test.get("call", {}).get("duration", 0), 3),
                })
            dur = round(data.get("duration", 0), 2)
            summary_line = (
                f"============================= "
                f"{counts['passed']} passed"
                + (f", {counts['failed']} failed" if counts["failed"] else "")
                + (f", {counts['error']} error"   if counts["error"]   else "")
                + f" in {dur}s ============================="
            )
        except Exception as e:
            print(f"  Warning: could not parse JSON report: {e}")

    print(f"\n  Log → {log_file}")
    print(f"  {summary_line or 'No summary available'}")

    return {
        "name":   suite["name"],
        "label":  suite["label"],
        "counts": counts,
        "tests":  test_rows,
        "summary_line": summary_line,
        "returncode": result.returncode,
    }


def main():
    print(f"\n{'#'*72}")
    print(f"  DocGenie API — Full Test Run")
    print(f"  {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'#'*72}")

    suite_results = [run_suite(s) for s in SUITES]

    # Save combined results for compile_results.py
    combined = {
        "generated": datetime.datetime.now().isoformat(),
        "suites": suite_results,
    }
    combined_file = ARTIFACTS / "combined_results.json"
    combined_file.write_text(json.dumps(combined, indent=2))

    # Print summary table
    print(f"\n\n{'='*72}")
    print("  OVERALL SUMMARY")
    print(f"{'='*72}")
    print(f"  {'Suite':<20} {'Total':>6} {'Passed':>7} {'Failed':>7} {'Errors':>7}")
    print(f"  {'-'*50}")
    all_pass = True
    for r in suite_results:
        c = r["counts"]
        ok = "✅" if r["returncode"] == 0 else "❌"
        print(f"  {ok} {r['name']:<18} {c['total']:>6} {c['passed']:>7} "
              f"{c['failed']:>7} {c['error']:>7}")
        if r["returncode"] != 0:
            all_pass = False

    print(f"\n  Results → {combined_file}")
    print(f"\n{'#'*72}")
    if all_pass:
        print("  🎉  All suites PASSED.")
    else:
        print("  ⚠️   Some suites FAILED — see logs in tests/artifacts/")
    print(f"{'#'*72}\n")

    return 0 if all_pass else 1


if __name__ == "__main__":
    sys.exit(main())